nikhil-kh commited on
Commit
80c8a75
Β·
1 Parent(s): 6a33bd3
Files changed (4) hide show
  1. app.py +101 -35
  2. rag.py +42 -25
  3. script_gen.py +46 -10
  4. url_loader.py +102 -0
app.py CHANGED
@@ -10,43 +10,88 @@ import gradio as gr
10
  from rag import build_vectorstore, retrieve_context
11
  from script_gen import generate_script
12
  from tts import script_to_audio
 
13
 
14
  GROQ_API_KEY = os.environ.get("GROQ_API_KEY", "")
15
  GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
16
 
 
17
 
18
- def run_pipeline(file, mode, topic_hint):
 
 
 
 
 
 
 
 
 
 
 
19
  groq_key = GROQ_API_KEY
20
  gemini_key = GEMINI_API_KEY
21
 
22
  if not groq_key and not gemini_key:
23
- yield ("❌ No API keys found. Add GROQ_API_KEY or GEMINI_API_KEY in Space Settings β†’ Secrets.", None, "")
24
  return
25
 
26
- if not file or (isinstance(file, list) and len(file) == 0):
27
- yield "❌ Please upload at least one document (PDF, TXT, or DOCX).", None, ""
 
 
 
 
 
 
28
  return
29
 
30
  try:
31
- n_files = len(file) if isinstance(file, list) else 1
32
- yield f"πŸ“š Reading {n_files} document(s) and building knowledge base...", None, ""
33
- vectorstore, _ = build_vectorstore(file)
34
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  query = topic_hint.strip() if topic_hint.strip() else f"main concepts key ideas themes {mode}"
36
- yield f"πŸ” Retrieving content for {mode} format...", None, ""
37
  context = retrieve_context(vectorstore, query, k=8, mode=mode)
38
 
 
39
  provider = "Groq (Llama 4)" if groq_key else "Gemini Flash"
40
- yield f"✍️ Generating {mode} script using {provider}...", None, ""
41
 
42
- script = generate_script(context=context, mode=mode, groq_key=groq_key, gemini_key=gemini_key)
 
 
 
 
43
 
44
  if not script.strip():
45
- yield "❌ Script generation returned empty output. Please try again.", None, ""
46
  return
47
 
48
- yield "πŸŽ™οΈ Script ready! Synthesizing audio, please wait...", None, script
49
 
 
50
  out_dir = tempfile.mkdtemp()
51
  unique_id = uuid.uuid4().hex[:8]
52
  audio_path = os.path.join(out_dir, f"voiceverse_{unique_id}.mp3")
@@ -67,17 +112,17 @@ def run_pipeline(file, mode, topic_hint):
67
  while not tts_result["done"]:
68
  time.sleep(3)
69
  dots = (dots % 3) + 1
70
- yield f"πŸŽ™οΈ Synthesizing audio{'.' * dots}", None, script
71
 
72
  if tts_result["error"]:
73
  raise ValueError(f"Audio synthesis failed: {tts_result['error']}")
74
 
75
- yield "⏳ Loading audio player...", audio_path, script
76
 
77
  except ValueError as e:
78
- yield f"❌ {str(e)}", None, ""
79
  except Exception as e:
80
- yield f"❌ Unexpected error: {str(e)}", None, ""
81
 
82
 
83
  CSS = """
@@ -98,6 +143,8 @@ MODE_CHOICES = [
98
  ("πŸŽ“ Lecture β€” Professor explains your document", "lecture"),
99
  ]
100
 
 
 
101
  with gr.Blocks(title="VoiceVerse – Document to Audio AI") as demo:
102
 
103
  gr.HTML("""
@@ -110,14 +157,21 @@ with gr.Blocks(title="VoiceVerse – Document to Audio AI") as demo:
110
 
111
  with gr.Row(equal_height=False):
112
 
 
113
  with gr.Column(scale=1, min_width=320):
114
- gr.Markdown("<p class='section-label'>Step 1 β€” Upload Documents</p>")
 
115
  file_input = gr.File(
116
- label="PDF, TXT, or DOCX (select multiple if needed)",
117
  file_types=[".pdf", ".txt", ".docx", ".doc"],
118
  file_count="multiple",
119
  type="filepath",
120
  )
 
 
 
 
 
121
 
122
  gr.Markdown("<p class='section-label'>Step 2 β€” Choose Format</p>")
123
  mode_select = gr.Radio(
@@ -126,7 +180,21 @@ with gr.Blocks(title="VoiceVerse – Document to Audio AI") as demo:
126
  value="podcast",
127
  )
128
 
129
- gr.Markdown("<p class='section-label'>Step 3 β€” Focus Topic (optional)</p>")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  topic_hint = gr.Textbox(
131
  label="What should the content focus on?",
132
  placeholder="e.g. 'key findings', 'chapter 2 concepts', or leave blank for full summary",
@@ -140,6 +208,7 @@ with gr.Blocks(title="VoiceVerse – Document to Audio AI") as demo:
140
  elem_id="generate-btn",
141
  )
142
 
 
143
  with gr.Column(scale=1, min_width=320):
144
  gr.Markdown("<p class='section-label'>Output</p>")
145
 
@@ -165,31 +234,28 @@ with gr.Blocks(title="VoiceVerse – Document to Audio AI") as demo:
165
  with gr.Accordion("ℹ️ How VoiceVerse Works", open=False):
166
  gr.Markdown("""
167
  **Pipeline:**
168
- 1. **Document Ingestion** β€” File split into ~600 char chunks with 100 char overlap (paragraph β†’ line β†’ sentence β†’ word)
169
- 2. **RAG Retrieval** β€” Chunks embedded with `sentence-transformers/all-MiniLM-L6-v2`, stored in FAISS. Each mode runs 3 targeted sub-queries β†’ different chunks β†’ different scripts from the same document.
170
- 3. **Script Generation** β€” Groq (Llama 4 Maverick) primary; Gemini 2.0 Flash automatic fallback.
171
- 4. **Neural TTS** β€” Microsoft Edge Neural voices with distinct voice + rate + pitch per speaker, stitched with natural pauses.
172
 
173
  **Voices:**
174
  | Format | Speaker | Voice |
175
  |---|---|---|
176
- | Podcast | ALEX | en-US-AndrewNeural (US Male, warm) |
177
- | Podcast | JAMIE | en-US-EmmaMultilingualNeural (US Female, natural) |
178
- | Debate | PRO | en-US-AriaNeural (US Female, expressive) |
179
- | Debate | CON | en-GB-RyanNeural (British Male, authoritative) |
180
- | Storytelling | NARRATOR | en-US-EmmaMultilingualNeural (slightly slower) |
181
- | Lecture | PROFESSOR | en-GB-RyanNeural (measured, academic) |
182
-
183
- *⚠️ All voices are synthetic AI-generated audio.*
184
  """)
185
 
186
  generate_btn.click(
187
  fn=run_pipeline,
188
- inputs=[file_input, mode_select, topic_hint],
189
- outputs=[status_box, audio_output, script_output],
190
  )
191
 
192
- # Fires when audio_output gets a new value β€” meaning browser has loaded it
193
  audio_output.change(
194
  fn=lambda a: "βœ… Done! Press play to listen." if a else "",
195
  inputs=[audio_output],
 
10
  from rag import build_vectorstore, retrieve_context
11
  from script_gen import generate_script
12
  from tts import script_to_audio
13
+ from url_loader import scrape_urls
14
 
15
  GROQ_API_KEY = os.environ.get("GROQ_API_KEY", "")
16
  GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
17
 
18
+ WORDS_PER_MINUTE = 130 # average spoken English pace
19
 
20
+
21
+ def recommend_duration(word_count: int) -> float:
22
+ """Recommend audio duration based on source word count."""
23
+ # ~20% of source words end up in the final script (summarisation factor)
24
+ estimated_script_words = word_count * 0.20
25
+ minutes = estimated_script_words / WORDS_PER_MINUTE
26
+ # Round to nearest 0.5, clamp between 1 and 10
27
+ minutes = max(1.0, min(10.0, round(minutes * 2) / 2))
28
+ return minutes
29
+
30
+
31
+ def run_pipeline(files, urls_text, mode, tone, duration, topic_hint):
32
  groq_key = GROQ_API_KEY
33
  gemini_key = GEMINI_API_KEY
34
 
35
  if not groq_key and not gemini_key:
36
+ yield "❌ No API keys found. Add GROQ_API_KEY or GEMINI_API_KEY in Space Settings β†’ Secrets.", None, "", gr.update()
37
  return
38
 
39
+ # Parse URLs β€” one per line
40
+ url_list = [u.strip() for u in (urls_text or "").splitlines() if u.strip()]
41
+
42
+ has_files = files and len(files) > 0
43
+ has_urls = len(url_list) > 0
44
+
45
+ if not has_files and not has_urls:
46
+ yield "❌ Please upload at least one document or add a URL.", None, "", gr.update()
47
  return
48
 
49
  try:
50
+ # ── Step 1: Scrape URLs ───────────────────────────────────────────────
51
+ url_texts = []
52
+ if has_urls:
53
+ yield f"🌐 Fetching {len(url_list)} URL(s)...", None, "", gr.update()
54
+ url_texts, url_errors = scrape_urls(url_list)
55
+ if url_errors and not url_texts and not has_files:
56
+ raise ValueError("All URLs failed:\n" + "\n".join(url_errors))
57
+ if url_errors:
58
+ print(f"[VoiceVerse] URL warnings: {url_errors}")
59
+
60
+ # ── Step 2: Build vectorstore ─────────────────────────────────────────
61
+ n_files = len(files) if has_files else 0
62
+ n_urls = len(url_texts)
63
+ yield f"πŸ“š Processing {n_files} file(s) and {n_urls} URL(s)...", None, "", gr.update()
64
+ vectorstore, _, word_count = build_vectorstore(
65
+ file_paths=files if has_files else None,
66
+ url_texts=url_texts if url_texts else None,
67
+ )
68
+
69
+ # Recommend duration based on actual source length
70
+ recommended = recommend_duration(word_count)
71
+ duration_update = gr.update(value=recommended)
72
+
73
+ # ── Step 3: Retrieve context ──────────────────────────────────────────
74
  query = topic_hint.strip() if topic_hint.strip() else f"main concepts key ideas themes {mode}"
75
+ yield f"πŸ” Retrieving content for {mode} format...", None, "", duration_update
76
  context = retrieve_context(vectorstore, query, k=8, mode=mode)
77
 
78
+ # ── Step 4: Generate script ───────────────────────────────────────────
79
  provider = "Groq (Llama 4)" if groq_key else "Gemini Flash"
80
+ yield f"✍️ Generating {tone.lower()} {mode} script (~{duration} min) using {provider}...", None, "", gr.update()
81
 
82
+ script = generate_script(
83
+ context=context, mode=mode,
84
+ groq_key=groq_key, gemini_key=gemini_key,
85
+ tone=tone, duration=duration,
86
+ )
87
 
88
  if not script.strip():
89
+ yield "❌ Script generation returned empty output. Please try again.", None, "", gr.update()
90
  return
91
 
92
+ yield "πŸŽ™οΈ Script ready! Synthesizing audio, please wait...", None, script, gr.update()
93
 
94
+ # ── Step 5: Synthesize audio ──────────────────────────────────────────
95
  out_dir = tempfile.mkdtemp()
96
  unique_id = uuid.uuid4().hex[:8]
97
  audio_path = os.path.join(out_dir, f"voiceverse_{unique_id}.mp3")
 
112
  while not tts_result["done"]:
113
  time.sleep(3)
114
  dots = (dots % 3) + 1
115
+ yield f"πŸŽ™οΈ Synthesizing audio{'.' * dots}", None, script, gr.update()
116
 
117
  if tts_result["error"]:
118
  raise ValueError(f"Audio synthesis failed: {tts_result['error']}")
119
 
120
+ yield "⏳ Loading audio player...", audio_path, script, gr.update()
121
 
122
  except ValueError as e:
123
+ yield f"❌ {str(e)}", None, "", gr.update()
124
  except Exception as e:
125
+ yield f"❌ Unexpected error: {str(e)}", None, "", gr.update()
126
 
127
 
128
  CSS = """
 
143
  ("πŸŽ“ Lecture β€” Professor explains your document", "lecture"),
144
  ]
145
 
146
+ TONE_CHOICES = ["Professional", "Casual", "Dramatic", "Humorous", "Educational"]
147
+
148
  with gr.Blocks(title="VoiceVerse – Document to Audio AI") as demo:
149
 
150
  gr.HTML("""
 
157
 
158
  with gr.Row(equal_height=False):
159
 
160
+ # ── LEFT: Inputs ──────────────────────────────────────────────────────
161
  with gr.Column(scale=1, min_width=320):
162
+
163
+ gr.Markdown("<p class='section-label'>Step 1 β€” Add Content</p>")
164
  file_input = gr.File(
165
+ label="Upload Documents (PDF, TXT, DOCX β€” select multiple)",
166
  file_types=[".pdf", ".txt", ".docx", ".doc"],
167
  file_count="multiple",
168
  type="filepath",
169
  )
170
+ urls_input = gr.Textbox(
171
+ label="And / Or paste URLs (one per line)",
172
+ placeholder="https://example.com/article\nhttps://another.com/page",
173
+ lines=3,
174
+ )
175
 
176
  gr.Markdown("<p class='section-label'>Step 2 β€” Choose Format</p>")
177
  mode_select = gr.Radio(
 
180
  value="podcast",
181
  )
182
 
183
+ gr.Markdown("<p class='section-label'>Step 3 β€” Tone & Duration</p>")
184
+ tone_select = gr.Dropdown(
185
+ label="Tone",
186
+ choices=TONE_CHOICES,
187
+ value="Professional",
188
+ )
189
+ duration_slider = gr.Slider(
190
+ label="Duration (minutes) β€” auto-recommended after upload",
191
+ minimum=1,
192
+ maximum=10,
193
+ step=0.5,
194
+ value=2.0,
195
+ )
196
+
197
+ gr.Markdown("<p class='section-label'>Step 4 β€” Focus Topic (optional)</p>")
198
  topic_hint = gr.Textbox(
199
  label="What should the content focus on?",
200
  placeholder="e.g. 'key findings', 'chapter 2 concepts', or leave blank for full summary",
 
208
  elem_id="generate-btn",
209
  )
210
 
211
+ # ── RIGHT: Outputs ────────────────────────────────────────────────────
212
  with gr.Column(scale=1, min_width=320):
213
  gr.Markdown("<p class='section-label'>Output</p>")
214
 
 
234
  with gr.Accordion("ℹ️ How VoiceVerse Works", open=False):
235
  gr.Markdown("""
236
  **Pipeline:**
237
+ 1. **Content Ingestion** β€” Upload PDFs/TXT/DOCX and/or paste URLs. URLs are scraped for clean text. All sources are merged into one knowledge base.
238
+ 2. **RAG Retrieval** β€” Chunks embedded with `sentence-transformers/all-MiniLM-L6-v2`, stored in FAISS. Each mode runs 3 targeted sub-queries for diverse, non-overlapping chunks.
239
+ 3. **Script Generation** β€” Groq (Llama 4 Maverick) primary; Gemini 2.0 Flash fallback. Tone and duration are injected into the prompt (~130 words/min).
240
+ 4. **Neural TTS** β€” Microsoft Edge Neural voices with distinct voice + rate + pitch per speaker.
241
 
242
  **Voices:**
243
  | Format | Speaker | Voice |
244
  |---|---|---|
245
+ | Podcast | ALEX | en-US-AndrewNeural |
246
+ | Podcast | JAMIE | en-US-EmmaMultilingualNeural |
247
+ | Debate | PRO | en-US-AriaNeural |
248
+ | Debate | CON | en-GB-RyanNeural |
249
+ | Storytelling | NARRATOR | en-US-EmmaMultilingualNeural |
250
+ | Lecture | PROFESSOR | en-GB-RyanNeural |
 
 
251
  """)
252
 
253
  generate_btn.click(
254
  fn=run_pipeline,
255
+ inputs=[file_input, urls_input, mode_select, tone_select, duration_slider, topic_hint],
256
+ outputs=[status_box, audio_output, script_output, duration_slider],
257
  )
258
 
 
259
  audio_output.change(
260
  fn=lambda a: "βœ… Done! Press play to listen." if a else "",
261
  inputs=[audio_output],
rag.py CHANGED
@@ -91,44 +91,60 @@ def load_document(file_path: str):
91
  return docs
92
 
93
 
94
- def build_vectorstore(file_paths):
95
  """
96
  Full RAG pipeline: load β†’ chunk β†’ embed β†’ index.
97
- Accepts a single file path (str) or a list of file paths.
98
- All files are merged into one vectorstore so retrieval
99
- works across the entire uploaded corpus.
100
- Returns (vectorstore, text_preview_string).
101
- """
102
- # Normalise to list
103
- if isinstance(file_paths, str):
104
- file_paths = [file_paths]
105
 
106
- # Filter out None entries (Gradio sometimes passes None)
107
- file_paths = [f for f in file_paths if f]
 
108
 
109
- if not file_paths:
110
- raise ValueError("No files provided. Please upload at least one document.")
 
 
 
111
 
112
  all_docs = []
113
  failed = []
114
 
115
- for fp in file_paths:
116
- try:
117
- docs = load_document(fp)
118
- all_docs.extend(docs)
119
- except Exception as e:
120
- failed.append(f"{Path(fp).name}: {str(e)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
 
122
  if failed and not all_docs:
123
- raise ValueError("All uploaded files failed to load:\n" + "\n".join(failed))
 
 
 
 
 
124
 
125
  if failed:
126
- print(f"[VoiceVerse RAG] Warning β€” some files failed: {failed}")
127
 
128
  full_text = " ".join([d.page_content for d in all_docs])
129
  if len(full_text.strip()) < 100:
130
  raise ValueError(
131
- "Documents have very little extractable text. "
132
  "Please check that files are not empty or image-only."
133
  )
134
 
@@ -140,12 +156,13 @@ def build_vectorstore(file_paths):
140
  chunks = splitter.split_documents(all_docs)
141
 
142
  if len(chunks) == 0:
143
- raise ValueError("Documents produced no chunks after splitting. They may be empty.")
144
 
145
  vectorstore = FAISS.from_documents(chunks, get_embeddings())
146
 
147
- preview = full_text[:300].strip()
148
- return vectorstore, preview
 
149
 
150
 
151
  def retrieve_context(vectorstore, query: str, k: int = 8, mode: str = None) -> str:
 
91
  return docs
92
 
93
 
94
+ def build_vectorstore(file_paths=None, url_texts=None):
95
  """
96
  Full RAG pipeline: load β†’ chunk β†’ embed β†’ index.
 
 
 
 
 
 
 
 
97
 
98
+ Accepts:
99
+ file_paths: str or list of file paths (PDF, TXT, DOCX)
100
+ url_texts: list of (url, text) tuples from url_loader.scrape_urls()
101
 
102
+ All sources are merged into one vectorstore so retrieval
103
+ works across all documents and URLs simultaneously.
104
+ Returns (vectorstore, text_preview_string, word_count).
105
+ """
106
+ from langchain.schema import Document
107
 
108
  all_docs = []
109
  failed = []
110
 
111
+ # ── Load files ────────────────────────────────────────────────────────────
112
+ if file_paths:
113
+ if isinstance(file_paths, str):
114
+ file_paths = [file_paths]
115
+ file_paths = [f for f in file_paths if f]
116
+
117
+ for fp in file_paths:
118
+ try:
119
+ docs = load_document(fp)
120
+ all_docs.extend(docs)
121
+ except Exception as e:
122
+ failed.append(f"{Path(fp).name}: {str(e)}")
123
+
124
+ # ── Load URL content ──────────────────────────────────────────────────────
125
+ if url_texts:
126
+ for url, text in url_texts:
127
+ if text and text.strip():
128
+ all_docs.append(Document(
129
+ page_content=text,
130
+ metadata={"source": url}
131
+ ))
132
 
133
  if failed and not all_docs:
134
+ raise ValueError("All sources failed to load:\n" + "\n".join(failed))
135
+
136
+ if not all_docs:
137
+ raise ValueError(
138
+ "No content found. Please upload at least one document or add a URL."
139
+ )
140
 
141
  if failed:
142
+ print(f"[VoiceVerse RAG] Warning β€” some sources failed: {failed}")
143
 
144
  full_text = " ".join([d.page_content for d in all_docs])
145
  if len(full_text.strip()) < 100:
146
  raise ValueError(
147
+ "Sources have very little extractable text. "
148
  "Please check that files are not empty or image-only."
149
  )
150
 
 
156
  chunks = splitter.split_documents(all_docs)
157
 
158
  if len(chunks) == 0:
159
+ raise ValueError("Sources produced no chunks after splitting.")
160
 
161
  vectorstore = FAISS.from_documents(chunks, get_embeddings())
162
 
163
+ word_count = len(full_text.split())
164
+ preview = full_text[:300].strip()
165
+ return vectorstore, preview, word_count
166
 
167
 
168
  def retrieve_context(vectorstore, query: str, k: int = 8, mode: str = None) -> str:
script_gen.py CHANGED
@@ -96,13 +96,29 @@ Rules:
96
  - All content must come from the source material""",
97
  }
98
 
 
 
 
 
 
 
 
 
99
  USER_PROMPT_TEMPLATE = """Create a complete, production-ready {mode} script based STRICTLY on the source material below.
100
 
 
 
 
 
 
 
 
101
  Critical rules:
102
  - Do NOT add any facts, claims, or details not explicitly present in the source
103
  - Keep all content grounded and accurate to the source
104
  - Write in natural spoken English β€” this will be converted to audio
105
- - The script must be at least 400 words long
 
106
  - Follow the format exactly as instructed (every line must have the correct speaker label)
107
 
108
  SOURCE MATERIAL:
@@ -113,7 +129,8 @@ Write the complete script now."""
113
 
114
  # ── Provider functions ────────────────────────────────────────────────────────
115
 
116
- def generate_with_groq(context: str, mode: str, api_key: str) -> str:
 
117
  """
118
  Generate script using Groq (Llama 4 Maverick).
119
  Free tier: 1,000 requests/day β€” no credit card needed.
@@ -121,6 +138,12 @@ def generate_with_groq(context: str, mode: str, api_key: str) -> str:
121
  """
122
  client = Groq(api_key=api_key)
123
 
 
 
 
 
 
 
124
  response = client.chat.completions.create(
125
  model="meta-llama/llama-4-maverick-17b-128e-instruct",
126
  messages=[
@@ -130,10 +153,14 @@ def generate_with_groq(context: str, mode: str, api_key: str) -> str:
130
  },
131
  {
132
  "role": "user",
133
- "content": USER_PROMPT_TEMPLATE.format(mode=mode, context=context),
 
 
 
 
134
  },
135
  ],
136
- max_tokens=2048,
137
  temperature=0.7,
138
  )
139
 
@@ -143,7 +170,8 @@ def generate_with_groq(context: str, mode: str, api_key: str) -> str:
143
  return result
144
 
145
 
146
- def generate_with_gemini(context: str, mode: str, api_key: str) -> str:
 
147
  """
148
  Generate script using Google Gemini 2.0 Flash.
149
  Free tier: ~20 requests/day β€” no credit card needed.
@@ -152,14 +180,18 @@ def generate_with_gemini(context: str, mode: str, api_key: str) -> str:
152
  """
153
  client = genai.Client(api_key=api_key)
154
 
155
- system = SYSTEM_PROMPTS.get(mode, SYSTEM_PROMPTS["podcast"])
156
- full_prompt = f"{system}\n\n{USER_PROMPT_TEMPLATE.format(mode=mode, context=context)}"
 
 
 
 
157
 
158
  response = client.models.generate_content(
159
  model="gemini-2.0-flash",
160
  contents=full_prompt,
161
  config=genai_types.GenerateContentConfig(
162
- max_output_tokens=2048,
163
  temperature=0.7,
164
  ),
165
  )
@@ -177,6 +209,8 @@ def generate_script(
177
  mode: str,
178
  groq_key: str = "",
179
  gemini_key: str = "",
 
 
180
  ) -> str:
181
  """
182
  Generate a spoken-word script from retrieved document context.
@@ -190,6 +224,8 @@ def generate_script(
190
  mode: One of 'podcast', 'debate', 'storytelling', 'lecture'
191
  groq_key: Groq API key (primary)
192
  gemini_key: Google Gemini API key (fallback)
 
 
193
 
194
  Returns:
195
  Generated script as a string with speaker labels on every line
@@ -199,7 +235,7 @@ def generate_script(
199
  # ── Try Groq first ────────────────────────────────────────────────────────
200
  if groq_key.strip():
201
  try:
202
- return generate_with_groq(context, mode, groq_key.strip())
203
  except Exception as e:
204
  errors.append(f"Groq error: {str(e)}")
205
  print(f"[VoiceVerse] Groq failed, trying Gemini. Reason: {e}")
@@ -207,7 +243,7 @@ def generate_script(
207
  # ── Fall back to Gemini ───────────────────────────────────────────────────
208
  if gemini_key.strip():
209
  try:
210
- return generate_with_gemini(context, mode, gemini_key.strip())
211
  except Exception as e:
212
  errors.append(f"Gemini error: {str(e)}")
213
  print(f"[VoiceVerse] Gemini also failed. Reason: {e}")
 
96
  - All content must come from the source material""",
97
  }
98
 
99
+ TONE_INSTRUCTIONS = {
100
+ "Professional": "Use formal, authoritative language. Be precise and clear. Avoid slang or casual expressions.",
101
+ "Casual": "Use relaxed, conversational language. Feel free to use everyday expressions and a friendly tone.",
102
+ "Dramatic": "Use expressive, emotive language. Build tension and emphasise key moments dramatically.",
103
+ "Humorous": "Use wit, light humour, and playful language. Keep it fun and engaging without being silly.",
104
+ "Educational": "Use clear, instructive language. Explain concepts patiently as if teaching someone new to the topic.",
105
+ }
106
+
107
  USER_PROMPT_TEMPLATE = """Create a complete, production-ready {mode} script based STRICTLY on the source material below.
108
 
109
+ Tone: {tone}
110
+ Tone instruction: {tone_instruction}
111
+
112
+ Duration target: Approximately {duration} minute(s) of spoken audio.
113
+ Word count target: Aim for approximately {target_words} words total.
114
+ (Spoken audio averages ~130 words per minute)
115
+
116
  Critical rules:
117
  - Do NOT add any facts, claims, or details not explicitly present in the source
118
  - Keep all content grounded and accurate to the source
119
  - Write in natural spoken English β€” this will be converted to audio
120
+ - Match the tone instruction strictly throughout the entire script
121
+ - Hit the word count target as closely as possible
122
  - Follow the format exactly as instructed (every line must have the correct speaker label)
123
 
124
  SOURCE MATERIAL:
 
129
 
130
  # ── Provider functions ────────────────────────────────────────────────────────
131
 
132
+ def generate_with_groq(context: str, mode: str, api_key: str,
133
+ tone: str = "Professional", duration: float = 2.0) -> str:
134
  """
135
  Generate script using Groq (Llama 4 Maverick).
136
  Free tier: 1,000 requests/day β€” no credit card needed.
 
138
  """
139
  client = Groq(api_key=api_key)
140
 
141
+ target_words = int(duration * 130)
142
+ tone_instruction = TONE_INSTRUCTIONS.get(tone, TONE_INSTRUCTIONS["Professional"])
143
+
144
+ # Scale max_tokens based on duration (avg ~1.5 tokens per word)
145
+ max_tokens = min(4096, max(1024, int(target_words * 1.6)))
146
+
147
  response = client.chat.completions.create(
148
  model="meta-llama/llama-4-maverick-17b-128e-instruct",
149
  messages=[
 
153
  },
154
  {
155
  "role": "user",
156
+ "content": USER_PROMPT_TEMPLATE.format(
157
+ mode=mode, context=context,
158
+ tone=tone, tone_instruction=tone_instruction,
159
+ duration=duration, target_words=target_words,
160
+ ),
161
  },
162
  ],
163
+ max_tokens=max_tokens,
164
  temperature=0.7,
165
  )
166
 
 
170
  return result
171
 
172
 
173
+ def generate_with_gemini(context: str, mode: str, api_key: str,
174
+ tone: str = "Professional", duration: float = 2.0) -> str:
175
  """
176
  Generate script using Google Gemini 2.0 Flash.
177
  Free tier: ~20 requests/day β€” no credit card needed.
 
180
  """
181
  client = genai.Client(api_key=api_key)
182
 
183
+ target_words = int(duration * 130)
184
+ tone_instruction = TONE_INSTRUCTIONS.get(tone, TONE_INSTRUCTIONS["Professional"])
185
+ max_tokens = min(4096, max(1024, int(target_words * 1.6)))
186
+
187
+ system = SYSTEM_PROMPTS.get(mode, SYSTEM_PROMPTS["podcast"])
188
+ full_prompt = f"{system}\n\n{USER_PROMPT_TEMPLATE.format(mode=mode, context=context, tone=tone, tone_instruction=tone_instruction, duration=duration, target_words=target_words)}"
189
 
190
  response = client.models.generate_content(
191
  model="gemini-2.0-flash",
192
  contents=full_prompt,
193
  config=genai_types.GenerateContentConfig(
194
+ max_output_tokens=max_tokens,
195
  temperature=0.7,
196
  ),
197
  )
 
209
  mode: str,
210
  groq_key: str = "",
211
  gemini_key: str = "",
212
+ tone: str = "Professional",
213
+ duration: float = 2.0,
214
  ) -> str:
215
  """
216
  Generate a spoken-word script from retrieved document context.
 
224
  mode: One of 'podcast', 'debate', 'storytelling', 'lecture'
225
  groq_key: Groq API key (primary)
226
  gemini_key: Google Gemini API key (fallback)
227
+ tone: One of 'Professional', 'Casual', 'Dramatic', 'Humorous', 'Educational'
228
+ duration: Target audio duration in minutes (e.g. 2.0 = ~260 words)
229
 
230
  Returns:
231
  Generated script as a string with speaker labels on every line
 
235
  # ── Try Groq first ────────────────────────────────────────────────────────
236
  if groq_key.strip():
237
  try:
238
+ return generate_with_groq(context, mode, groq_key.strip(), tone, duration)
239
  except Exception as e:
240
  errors.append(f"Groq error: {str(e)}")
241
  print(f"[VoiceVerse] Groq failed, trying Gemini. Reason: {e}")
 
243
  # ── Fall back to Gemini ───────────────────────────────────────────────────
244
  if gemini_key.strip():
245
  try:
246
+ return generate_with_gemini(context, mode, gemini_key.strip(), tone, duration)
247
  except Exception as e:
248
  errors.append(f"Gemini error: {str(e)}")
249
  print(f"[VoiceVerse] Gemini also failed. Reason: {e}")
url_loader.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # url_loader.py β€” Scrape web pages and return clean plain text
2
+ # Uses requests + BeautifulSoup β€” free, no API key needed.
3
+
4
+ import requests
5
+ from bs4 import BeautifulSoup
6
+
7
+
8
+ HEADERS = {
9
+ "User-Agent": (
10
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
11
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
12
+ "Chrome/120.0.0.0 Safari/537.36"
13
+ )
14
+ }
15
+
16
+ TIMEOUT = 15 # seconds
17
+
18
+
19
+ def scrape_url(url: str) -> str:
20
+ """
21
+ Fetch a URL and return clean plain text.
22
+ Strips nav, header, footer, scripts, and ads.
23
+ Raises ValueError with a user-friendly message on failure.
24
+ """
25
+ url = url.strip()
26
+ if not url:
27
+ raise ValueError("Empty URL provided.")
28
+
29
+ if not url.startswith(("http://", "https://")):
30
+ url = "https://" + url
31
+
32
+ try:
33
+ response = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
34
+ response.raise_for_status()
35
+ except requests.exceptions.Timeout:
36
+ raise ValueError(f"Timed out trying to reach: {url}")
37
+ except requests.exceptions.ConnectionError:
38
+ raise ValueError(f"Could not connect to: {url}. Check the URL and try again.")
39
+ except requests.exceptions.HTTPError as e:
40
+ raise ValueError(f"HTTP error {e.response.status_code} for: {url}")
41
+ except Exception as e:
42
+ raise ValueError(f"Failed to fetch {url}: {str(e)}")
43
+
44
+ soup = BeautifulSoup(response.content, "html.parser")
45
+
46
+ # Remove noisy elements
47
+ for tag in soup(["script", "style", "nav", "header", "footer",
48
+ "aside", "form", "noscript", "iframe", "ads",
49
+ "advertisement", "cookie", "popup"]):
50
+ tag.decompose()
51
+
52
+ # Also remove by common class/id patterns for ads and nav
53
+ for tag in soup.find_all(True, {"class": [
54
+ "nav", "navbar", "menu", "sidebar", "footer", "header",
55
+ "advertisement", "ad", "cookie", "popup", "banner"
56
+ ]}):
57
+ tag.decompose()
58
+
59
+ # Get text from the body
60
+ body = soup.find("body") or soup
61
+ text = body.get_text(separator="\n")
62
+
63
+ # Clean up whitespace
64
+ lines = [line.strip() for line in text.splitlines()]
65
+ lines = [line for line in lines if len(line) > 30] # drop short/empty lines
66
+ clean_text = "\n".join(lines)
67
+
68
+ if len(clean_text.strip()) < 100:
69
+ raise ValueError(
70
+ f"Could not extract meaningful text from: {url}\n"
71
+ "The page may require JavaScript, be paywalled, or block scrapers."
72
+ )
73
+
74
+ return clean_text
75
+
76
+
77
+ def scrape_urls(urls: list) -> list:
78
+ """
79
+ Scrape multiple URLs. Returns list of (url, text) tuples for successes.
80
+ Collects errors without crashing β€” returns them separately.
81
+
82
+ Returns:
83
+ (results, errors)
84
+ results: list of (url, text) for successful scrapes
85
+ errors: list of error strings for failed scrapes
86
+ """
87
+ results = []
88
+ errors = []
89
+
90
+ for url in urls:
91
+ url = url.strip()
92
+ if not url:
93
+ continue
94
+ try:
95
+ text = scrape_url(url)
96
+ results.append((url, text))
97
+ print(f"[VoiceVerse URL] βœ“ Scraped {url} ({len(text)} chars)")
98
+ except ValueError as e:
99
+ errors.append(str(e))
100
+ print(f"[VoiceVerse URL] βœ— Failed {url}: {e}")
101
+
102
+ return results, errors