File size: 7,880 Bytes
8422173 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | import os
import time
import json
import keyboard # pip install keyboard==0.13.5
import numpy as np
import faiss
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed
# Import the AudioProcessor class from your STT module.
from stt import AudioProcessor
from llm import LLMProcessor
# Import vectorstore functions from vectorstore.py (located in app/)
from vectorstore import (
update_vectorstore_from_pdf,
save_vectorstore_with_timestamp,
load_vectorstore,
search_vectorstore
)
# Import TTS functions from the tts folder.
from tts import chunk_text_for_tts, synthesize_speech_guideline, play_audio
def combine_vectorstores(index1, chunks1, index2, chunks2):
"""
Combine two FAISS indexes (both IndexFlatL2) by retrieving all embeddings,
concatenating them, and building a new FAISS index.
Also combines the two lists of text chunks.
"""
embeddings1 = np.array([index1.reconstruct(i) for i in range(index1.ntotal)])
embeddings2 = np.array([index2.reconstruct(i) for i in range(index2.ntotal)])
combined_embeddings = np.concatenate([embeddings1, embeddings2], axis=0)
combined_chunks = chunks1 + chunks2
dimension = combined_embeddings.shape[1]
combined_index = faiss.IndexFlatL2(dimension)
combined_index.add(combined_embeddings)
return combined_index, combined_chunks
def build_and_save_vectorstore():
"""
Process two PDFs, combine their embeddings into one vectorstore,
save the combined index with a timestamp, and return the index,
chunks, and embedder used.
"""
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
pdf_path1 = os.path.join(base_dir, "data", "PDF", "cancer_dictionary.pdf")
pdf_path2 = os.path.join(base_dir, "data", "PDF", "Medical Dictionary.pdf")
index1, chunks1, embedder = update_vectorstore_from_pdf(pdf_path1, chunk_size=600)
index2, chunks2, _ = update_vectorstore_from_pdf(pdf_path2, chunk_size=600)
combined_index, combined_chunks = combine_vectorstores(index1, chunks1, index2, chunks2)
saved_index_path = save_vectorstore_with_timestamp(combined_index)
print(f"Combined vectorstore saved at: {saved_index_path}")
loaded_index = load_vectorstore(saved_index_path)
return loaded_index, combined_chunks, embedder
def synthesize_audio_chunks(text, max_chars=300):
"""
Breaks the given text into chunks and asynchronously synthesizes them.
Yields the output file for each synthesized audio chunk.
"""
if not text:
raise ValueError("No text provided for TTS synthesis.")
# Break the text into manageable chunks.
chunks = chunk_text_for_tts(text, max_chars=max_chars)
print("TTS Text chunks:")
for i, chunk in enumerate(chunks, start=1):
print(f"Chunk {i}: {chunk}\n")
# Use a ThreadPoolExecutor to process the chunks concurrently.
with ThreadPoolExecutor(max_workers=4) as executor:
# Submit synthesis tasks and keep a mapping of future to its chunk index.
future_to_index = {
executor.submit(synthesize_speech_guideline, chunk, output_file=f"audio_chunk_{i}.wav"): i
for i, chunk in enumerate(chunks, start=1)
}
# Yield audio files as soon as each future completes.
for future in as_completed(future_to_index):
index = future_to_index[future]
try:
audio_file = future.result()
if audio_file is None:
print(f"Warning: Synthesis for chunk {index} returned None.")
else:
print(f"Chunk {index} synthesized successfully: {audio_file}")
yield audio_file
except Exception as exc:
print(f"Error synthesizing chunk {index}: {exc}")
def synthesize_audio_chunks(text, max_chars=300):
"""
Breaks the given text into chunks and asynchronously synthesizes them.
Yields the output file for each synthesized audio chunk.
"""
if not text:
raise ValueError("No text provided for TTS synthesis.")
chunks = chunk_text_for_tts(text, max_chars=max_chars)
print("TTS Text chunks:")
for i, chunk in enumerate(chunks, 1):
print(f"Chunk {i}: {chunk}\n")
with ThreadPoolExecutor(max_workers=4) as executor:
# Submit synthesis tasks for each chunk
futures = [executor.submit(synthesize_speech_guideline, chunk, output_file=f"audio_chunk_{i}.wav")
for i, chunk in enumerate(chunks)]
# Yield audio files as soon as each synthesis task completes
for future in futures:
audio_file = future.result()
yield audio_file
def main():
print("Building vectorstore from PDFs...")
vector_index, vector_chunks, embedder = build_and_save_vectorstore()
processor = AudioProcessor()
llm_processor = LLMProcessor()
print("\nLooping process: Use push-to-talk to record and transcribe.")
print("Press 's' at any time to stop the program.")
Saved_response = ""
while True:
if keyboard.is_pressed('s'):
print("Stop key pressed. Exiting program.")
break
print("\nReady to record. Press 'q' to start and 'q' again to stop recording.\n----------------\n")
denoised_audio, fs = processor.record_and_denoise()
if denoised_audio is None:
print("No audio was recorded. Skipping transcription.")
else:
print("Transcribing audio...")
try:
transcription = processor.transcribe_audio(processor.default_output_file, language="en")
print("Transcribed Text:\n", transcription)
search_results = search_vectorstore(transcription, embedder, vector_index, vector_chunks, top_k=3)
query = Saved_response + transcription
for i, result in enumerate(search_results, 1):
query += f"\n{i}. {result}"
try:
gemini_response = llm_processor.call_gemini_llm("gemini-2.0-flash", query)
print("Gemini response:", gemini_response)
except Exception as e:
print("Gemini error:", e)
gemini_response = f"Error: {str(e)}"
Saved_response += "transcription: " + transcription + "\n" + "gemini_response: " + gemini_response + "\n"
processor.log_conversation(transcription, bot_text=gemini_response)
# Synthesize and play TTS output in a pipelined fashion.
print("Synthesizing and playing LLM response via TTS...")
print()
audio_generator = synthesize_audio_chunks(gemini_response, max_chars=300)
# Immediately get the first audio chunk and play it
try:
first_audio = next(audio_generator)
play_audio(first_audio)
except StopIteration:
print("No audio chunks generated.")
# Now play the rest of the chunks as soon as they are ready.
for audio_file in audio_generator:
play_audio(audio_file)
except Exception as e:
print("Error during transcription:", e)
print("Iteration complete. Waiting 5 seconds before next recording...")
time.sleep(5)
print("Program terminated.")
if __name__ == "__main__":
main() |