Voice assistant improvements

.gitignore

@@ -47,5 +47,6 @@ src/main.py
 src/chat.py
 src/llm/prompt2.py
 src/loaders
+/notebook/
 
 ###

README.md

@@ -9,7 +9,6 @@ app_file: app.py
 pinned: false
 ---
 
-
 # RAG Assistant
 
 Assistant conversationnel basé sur une architecture **Retrieval Augmented Generation (RAG)**.

app.py

@@ -1,7 +1,8 @@
 import spaces
-
 import gradio as gr
 
+from pathlib import Path
+
 from src.vectorstore.faiss_index import load_index
 from src.vectorstore.metadata_store import load_metadata
 
@@ -13,35 +14,63 @@ from src.llm.postprocessing import clean_answer
 
 from src.data_processing.txt_embeddings import compute_embeddings
 
-# Load once
-from pathlib import Path
+from src.voice.assistant import text_to_audio
+from src.voice.speech_to_text import transcribe_audio
+
+
+# ============================================================
+# Chargement des ressources
+# ============================================================
 
 BASE_DIR = Path(__file__).parent
 
 INDEX_PATH = str(BASE_DIR / "data/vectorstore/faiss.index")
 METADATA_PATH = str(BASE_DIR / "data/metadata/metadata.json")
 
+
 index = load_index(INDEX_PATH)
 metadata = load_metadata(METADATA_PATH)
 
-@spaces.GPU
 
+# ============================================================
+# Fonction RAG + TTS
+# ============================================================
+@spaces.GPU
 
+#
 def chat_fn(message, history):
 
-    # 1. Embedding de la question
+    # 1. Embedding question
     query_emb = compute_embeddings([message])
 
+
     # 2. Retrieval FAISS
-    docs = retrieve(query_emb, index, metadata, top_k=10)
+    docs = retrieve(
+        query_emb,
+        index,
+        metadata,
+        top_k=10
+    )
+
 
-    # 3. Reranking (IMPORTANT: retourne (doc, score))
-    docs = rerank(message, docs, top_k=10)
+    # 3. Reranking
+    docs = rerank(
+        message,
+        docs,
+        top_k=4
+    )
 
-    # 4. Construction du contexte LLM
-    context = "\n\n".join([doc["text"] for doc, score in docs])
 
-    # 5. Prompt
+    # 4. Construction contexte
+    context = "\n\n".join(
+        [
+            doc["text"]
+            for doc, score in docs
+        ]
+    )
+
+
+    # 5. Prompt LLM
     prompt = f"""
 Context:
 {context}
@@ -51,14 +80,20 @@ Question:
 
 Answer:
 """
-
-    # 6. Génération réponse LLM
+    # 6. Génération réponse
     answer = generate_answer(prompt)
 
-    # 6.1. Postprocessing
+    # 7. Nettoyage
     answer = clean_answer(answer)
 
-    # 7. Debug chunks avec scores
+    # 8. Génération audio Kokoro
+
+    try:
+        audio = text_to_audio(answer)
+    except Exception:
+        audio = None
+
+    # Debug sources
     debug = "\n\n".join(
         [
             f"📄 Chunk {i+1} (Score : {score:.3f})\n{doc['text']}"
@@ -66,19 +101,179 @@ Answer:
         ]
     )
 
-    # 8. Output final
-    final_output = f"{answer}\n\n---\n\n🔍 Retrieved chunks:\n{debug}"
 
-    return final_output
+    final_output = (
+        f"{answer}\n\n"
+        "---\n\n"
+        f"🔍 Retrieved chunks:\n{debug}"
+    )
+
+    return final_output, audio
+
+#
+def respond_audio(audio, history):
+
+    print("respond_audio appelée")
+    print(audio)
+
+    if audio is None:
+        return history, None
+
+    if history is None:
+        history = []
+
+    message = transcribe_audio(audio)
+
+    answer, output_audio = chat_fn(
+        message,
+        history
+    )
+
+    history.append(
+        {
+            "role": "user",
+            "content": message
+        }
+    )
+
+    history.append(
+        {
+            "role": "assistant",
+            "content": answer
+        }
+    )
+
+    return history, output_audio
+
+# ============================================================
+# Interface Gradio
+# ============================================================
+
+with gr.Blocks() as demo:
 
-###########################################################################
-# Interface Gradio type ChatGPT
+    gr.Markdown(
+        "# 💬 RAG Assistant vocal"
+    )
+
+
+    # ========================================================
+    # Ligne 1 : Chatbot
+    # ========================================================
+
+    chatbot = gr.Chatbot(
+        label="Conversation",
+        height=450
+    )
+
+
+    # ========================================================
+    # Ligne 2 : Question texte
+    # ========================================================
+
+    msg = gr.Textbox(
+        placeholder="Pose ta question...",
+        label="✍️ Question texte"
+    )
+
+
+    # ========================================================
+    # Ligne 3 : Audio entrée / sortie
+    # ========================================================
+
+    with gr.Row():
+
+        with gr.Column():
+
+            audio_input = gr.Audio(
+                sources=[
+                    "microphone",
+                    "upload"
+                ],
+                type="filepath",
+                waveform_options=gr.WaveformOptions(
+                    show_recording_waveform=True
+                ),
+                label="🎤 Parlez ou déposez un fichier audio"
+            )
+
+
+        with gr.Column():
+
+            audio_output = gr.Audio(
+                label="🔊 Réponse audio"
+            )
+
+
+    # ========================================================
+    # Réponse texte
+    # ========================================================
+
+    def respond(message, history):
+
+        if history is None:
+            history = []
+
+        answer, audio = chat_fn(
+            message,
+            history
+        )
+
+
+        history.append(
+            {
+                "role": "user",
+                "content": message
+            }
+        )
+
+
+        history.append(
+            {
+                "role": "assistant",
+                "content": answer
+            }
+        )
+
+
+        return (
+            "",
+            history,
+            audio
+        )
+
+    # ========================================================
+    # Events
+    # ========================================================
+
+    msg.submit(
+        respond,
+        inputs=[
+            msg,
+            chatbot
+        ],
+        outputs=[
+            msg,
+            chatbot,
+            audio_output
+        ]
+    )
+
+
+    audio_input.change(
+        respond_audio,
+        inputs=[
+            audio_input,
+            chatbot
+        ],
+        outputs=[
+            chatbot,
+            audio_output
+        ]
+    )
 
-demo = gr.ChatInterface(
-    fn=chat_fn,
-    title="💬 RAG Assistant",
-    description="Chat avec ton système RAG local"
-)
+# ============================================================
+# Run
+# ============================================================
 
 if __name__ == "__main__":
     demo.launch()

app_back.py

@@ -0,0 +1,98 @@
+import spaces
+
+import gradio as gr
+
+from src.vectorstore.faiss_index import load_index
+from src.vectorstore.metadata_store import load_metadata
+
+from src.retrieval.retriever import retrieve
+from src.retrieval.reranker import rerank
+
+from src.llm.generator import generate_answer
+from src.llm.postprocessing import clean_answer
+
+from src.data_processing.txt_embeddings import compute_embeddings
+
+from src.voice.assistant import text_to_audio
+
+# Load once
+from pathlib import Path
+
+BASE_DIR = Path(__file__).parent
+
+INDEX_PATH = str(BASE_DIR / "data/vectorstore/faiss.index")
+METADATA_PATH = str(BASE_DIR / "data/metadata/metadata.json")
+
+index = load_index(INDEX_PATH)
+metadata = load_metadata(METADATA_PATH)
+
+@spaces.GPU
+
+
+def chat_fn(message, history):
+
+    # 1. Embedding de la question
+    query_emb = compute_embeddings([message])
+
+    # 2. Retrieval FAISS
+    docs = retrieve(query_emb, index, metadata, top_k=10)
+
+    # 3. Reranking (IMPORTANT: retourne (doc, score))
+    docs = rerank(message, docs, top_k=10)
+
+    # 4. Construction du contexte LLM
+    context = "\n\n".join([doc["text"] for doc, score in docs])
+
+    # 5. Prompt
+    prompt = f"""
+Context:
+{context}
+
+Question:
+{message}
+
+Answer:
+"""
+
+    # 6. Génération réponse LLM
+    answer = generate_answer(prompt)
+
+    # 6.1. Postprocessing
+    answer = clean_answer(answer)
+
+    # 6.2. Audio answer
+    audio = text_to_audio(answer)
+
+    # 7. Debug chunks avec scores
+    debug = "\n\n".join(
+        [
+            f"📄 Chunk {i+1} (Score : {score:.3f})\n{doc['text']}"
+            for i, (doc, score) in enumerate(docs)
+        ]
+    )
+
+    # 8. Output final
+    final_output = f"{answer}\n\n---\n\n🔍 Retrieved chunks:\n{debug}"
+
+
+    return final_output, audio
+
+###########################################################################
+# Interface Gradio type ChatGPT
+
+"""
+
+audio_output = gr.Audio(
+    label="🔊 Audio"
+)
+
+demo = gr.ChatInterface(
+    fn=chat_fn,
+    title="💬 RAG Assistant",
+    description="Chat avec ton système RAG local",
+    additional_outputs=[audio_output]
+)
+
+if __name__ == "__main__":
+    demo.launch()
+"""

requirements.txt

@@ -1,7 +1,7 @@
-torch==2.12.1
+torch
 transformers==5.12.1
 sentence-transformers==5.6.0
-numpy==2.4.6
+numpy
 nltk==3.9.4
 python-docx==1.2.0
 PyMuPDF==1.28.0
@@ -10,4 +10,6 @@ faiss-cpu==1.14.3
 accelerate==1.14.0
 safetensors==0.8.0
 gradio==6.19.0
-spaces==0.50.4
+spaces==0.50.4
+kokoro==0.9.4
+faster_whisper==1.2.1

src/voice/assistant.py

@@ -0,0 +1,30 @@
+from kokoro import KPipeline
+import numpy as np
+
+
+# Chargement unique du modèle
+pipeline = KPipeline(lang_code="f")
+
+VOICE = "ff_siwis"
+SAMPLE_RATE = 24000
+
+
+def text_to_audio(text):
+    """
+    Convertit un texte en audio.
+    Retourne un tuple compatible avec gr.Audio
+    """
+
+    generator = pipeline(
+        text,
+        voice=VOICE
+    )
+
+    chunks = []
+
+    for _, _, audio in generator:
+        chunks.append(audio)
+
+    audio = np.concatenate(chunks)
+
+    return SAMPLE_RATE, audio

src/voice/speech_to_text.py

@@ -0,0 +1,42 @@
+from faster_whisper import WhisperModel
+import torch
+
+
+print("CUDA disponible :", torch.cuda.is_available())
+print("Nombre de GPU :", torch.cuda.device_count())
+
+
+if torch.cuda.is_available():
+    print("GPU :", torch.cuda.get_device_name(0))
+    device = "cuda"
+    compute_type = "float16"
+
+else:
+    device = "cpu"
+    compute_type = "int8"
+
+
+print("DEVICE :", device)
+print("COMPUTE :", compute_type)
+
+
+model = WhisperModel(
+    "base",
+    device=device,
+    compute_type=compute_type
+)
+
+
+def transcribe_audio(audio_path):
+
+    segments, info = model.transcribe(
+        audio_path,
+        language="fr"
+    )
+
+    text = " ".join(
+        segment.text
+        for segment in segments
+    )
+
+    return text.strip()