lization commited on
Commit
049c50e
ยท
verified ยท
1 Parent(s): 46498b6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +189 -131
app.py CHANGED
@@ -9,7 +9,7 @@ RAG Pipeline (6 steps):
9
  2. CHUNK โ†’ LangChain splits each article into overlapping 400-char chunks
10
  3. EMBED โ†’ Sentence Transformers encodes every chunk as a dense vector
11
  4. STORE โ†’ ChromaDB holds all vectors in an in-memory collection
12
- 5. RETRIEVE โ†’ Semantic search returns the 10 most relevant chunks
13
  6. GENERATE โ†’ Gemini Flash analyses the retrieved context and returns a structured signal
14
 
15
  Built by: Sammie Wong
@@ -18,33 +18,55 @@ Built by: Sammie Wong
18
  import os
19
  import json
20
  import uuid
 
21
  import gradio as gr
22
  import google.generativeai as genai
23
  import chromadb
24
  from chromadb.utils import embedding_functions
25
  from newsapi import NewsApiClient
26
- from langchain_text_splitters import RecursiveCharacterTextSplitter
27
  from datetime import datetime, timedelta
28
 
29
 
30
  # โ”€โ”€โ”€ STEP 1: FETCH โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
31
 
32
- def fetch_articles(topic: str, days_back: int, newsapi_key: str) -> list[dict]:
 
33
  """
34
- Pull recent news articles from NewsAPI for the given topic.
35
- Free tier: up to 100 requests/day, articles from the last 30 days.
 
 
 
 
 
 
 
 
 
36
  """
37
  client = NewsApiClient(api_key=newsapi_key)
38
  from_date = (datetime.now() - timedelta(days=days_back)).strftime('%Y-%m-%d')
39
 
40
- response = client.get_everything(
41
- q=f'"{topic}"', # Exact phrase match for cleaner signal
42
  language='en',
43
  sort_by='publishedAt',
44
  from_param=from_date,
45
- page_size=50 # NewsAPI free tier maximum
46
  )
47
- return response.get('articles', [])
 
 
 
 
 
 
 
 
 
 
 
 
48
 
49
 
50
  # โ”€โ”€โ”€ STEP 2: CHUNK โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
@@ -58,18 +80,17 @@ def chunk_articles(articles: list[dict]) -> tuple[list[str], list[dict]]:
58
  losing meaning at chunk boundaries.
59
  """
60
  splitter = RecursiveCharacterTextSplitter(
61
- chunk_size=400, # ~80 words โ€” enough context per chunk
62
- chunk_overlap=40 # Overlap prevents boundary information loss
63
  )
64
 
65
  all_chunks, all_metadata = [], []
66
 
67
  for article in articles:
68
- # Combine title + description + content for the richest signal
69
  full_text = " ".join(filter(None, [
70
  article.get('title', ''),
71
  article.get('description', ''),
72
- article.get('content', '')
73
  ]))
74
 
75
  for chunk in splitter.split_text(full_text):
@@ -77,7 +98,7 @@ def chunk_articles(articles: list[dict]) -> tuple[list[str], list[dict]]:
77
  all_metadata.append({
78
  'source': article['source']['name'],
79
  'published_at': article['publishedAt'][:10],
80
- 'title': (article.get('title') or '')[:100]
81
  })
82
 
83
  return all_chunks, all_metadata
@@ -91,31 +112,29 @@ def build_vector_store(chunks: list[str], metadatas: list[dict]):
91
  in an in-memory ChromaDB collection.
92
 
93
  Model: all-MiniLM-L6-v2 โ€” fast, accurate, no GPU needed.
94
- Storage: EphemeralClient (in-memory) means no disk writes โ€” perfect for
95
- a serverless HF Space where each request is stateless.
96
  """
97
  embedding_fn = embedding_functions.SentenceTransformerEmbeddingFunction(
98
  model_name="all-MiniLM-L6-v2"
99
  )
100
 
101
  try:
102
- db = chromadb.EphemeralClient() # ChromaDB >= 0.4.x
103
  except AttributeError:
104
- db = chromadb.Client() # Fallback for older versions
105
 
106
  collection = db.create_collection(
107
  name=f"trend_{uuid.uuid4().hex[:8]}",
108
- embedding_function=embedding_fn
109
  )
110
 
111
- # Insert in batches of 100 to avoid memory spikes
112
  batch_size = 100
113
  for i in range(0, len(chunks), batch_size):
114
- batch_end = i + batch_size
115
  collection.add(
116
- documents=chunks[i:batch_end],
117
- metadatas=metadatas[i:batch_end],
118
- ids=[f"c{i + j}" for j in range(len(chunks[i:batch_end]))]
119
  )
120
 
121
  return collection
@@ -123,50 +142,64 @@ def build_vector_store(chunks: list[str], metadatas: list[dict]):
123
 
124
  # โ”€โ”€โ”€ STEP 5: RETRIEVE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
125
 
126
- def retrieve_context(collection, topic: str, n_results: int = 10) -> tuple[list[str], list[dict]]:
127
  """
128
- Run a semantic similarity search to retrieve the most relevant chunks.
129
 
130
- The query is phrased to surface trend-signal language โ€” not just
131
- topic mentions. ChromaDB uses cosine similarity on the embeddings.
132
  """
133
  query = f"trend momentum growth decline viral popularity media coverage {topic}"
134
 
135
  results = collection.query(
136
  query_texts=[query],
137
- n_results=min(n_results, collection.count())
138
  )
139
  return results['documents'][0], results['metadatas'][0]
140
 
141
 
142
  # โ”€โ”€โ”€ STEP 6: GENERATE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
143
 
 
 
 
 
 
144
  def analyse_with_gemini(
145
  topic: str,
146
  retrieved_docs: list[str],
147
  retrieved_meta: list[dict],
148
  total_articles: int,
149
  days_back: int,
150
- gemini_key: str
151
  ) -> dict:
152
  """
153
- Pass the retrieved chunks (NOT all articles) to Gemini Flash for analysis.
154
-
155
- This is the core RAG advantage: the model only sees the most semantically
156
- relevant evidence, which produces more focused and accurate analysis
157
- than dumping all 50 articles into the prompt.
158
 
159
- response_mime_type="application/json" tells Gemini to return clean JSON
160
- directly โ€” no markdown fences, no preamble, no parsing headaches.
 
 
161
  """
162
- context = "\n\n".join([
163
- f"[{m['source']} ยท {m['published_at']}]\n{doc}"
164
- for doc, m in zip(retrieved_docs, retrieved_meta)
165
- ])
 
 
 
 
 
 
 
 
 
 
 
166
 
167
  prompt = f"""You are a senior trend analyst at a social listening company (like Pulsar, Brandwatch, or Sprinklr).
168
 
169
- You have been given the {len(retrieved_docs)} most semantically relevant news excerpts about "{topic}",
170
  retrieved via RAG from a corpus of {total_articles} articles published in the last {days_back} days.
171
 
172
  RETRIEVED CONTEXT:
@@ -200,104 +233,120 @@ Phase definitions:
200
  model = genai.GenerativeModel(
201
  model_name="gemini-2.0-flash",
202
  generation_config=genai.GenerationConfig(
203
- response_mime_type="application/json", # Forces clean JSON output
204
- max_output_tokens=900,
205
- )
206
  )
207
- response = model.generate_content(prompt)
208
- return json.loads(response.text)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209
 
210
 
211
  # โ”€โ”€โ”€ PIPELINE ORCHESTRATOR โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
212
 
213
- def run_analysis(topic: str, days_back: int, newsapi_key_input: str, gemini_key_input: str):
 
214
  """
215
- Runs the full 6-step RAG pipeline and returns formatted Gradio outputs.
216
- Returns 6 values matching the 6 gr.Markdown output components.
217
  """
218
  EMPTY = ("", "", "", "", "", "")
219
 
220
  if not topic.strip():
221
- return ("โš ๏ธ Please enter a topic.", *EMPTY[1:])
222
 
223
- # Resolve API keys: UI input โ†’ environment variable โ†’ error
224
  news_key = newsapi_key_input.strip() or os.environ.get("NEWSAPI_KEY", "")
225
  gemini_key = gemini_key_input.strip() or os.environ.get("GEMINI_API_KEY", "")
226
 
227
  if not news_key:
228
- return ("โš ๏ธ NewsAPI key is required. Get a free key at newsapi.org", *EMPTY[1:])
229
  if not gemini_key:
230
- return ("โš ๏ธ Gemini API key is required. Get a free key at aistudio.google.com", *EMPTY[1:])
 
 
 
231
 
232
  try:
233
- # โ”€โ”€ Step 1: Fetch โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
234
- articles = fetch_articles(topic, int(days_back), news_key)
235
  if not articles:
236
- return (
237
- f"โš ๏ธ No articles found for **'{topic}'** in the last {days_back} days.\n\n"
238
- "Try a broader topic, shorter name, or longer date range.",
239
- *EMPTY[1:]
240
  )
 
241
 
242
- # โ”€โ”€ Steps 2โ€“4: Chunk โ†’ Embed โ†’ Store โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
243
  chunks, metadatas = chunk_articles(articles)
244
  collection = build_vector_store(chunks, metadatas)
245
 
246
- # โ”€โ”€ Step 5: Retrieve โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
247
  retrieved_docs, retrieved_meta = retrieve_context(collection, topic)
248
 
249
- # โ”€โ”€ Step 6: Generate โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
250
  result = analyse_with_gemini(
251
  topic, retrieved_docs, retrieved_meta,
252
- len(articles), int(days_back), gemini_key
253
  )
254
 
255
- # โ”€โ”€ Format outputs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
256
- phase_icons = {
257
- "Early": "๐ŸŒฑ", "Rising": "๐Ÿ“ˆ", "Peak": "๐Ÿ”ฅ",
258
- "Declining": "๐Ÿ“‰", "Fading": "๐ŸŒซ๏ธ"
259
- }
260
- phase_colors = {
261
- "Early": "#22c55e", "Rising": "#3b82f6", "Peak": "#f97316",
262
- "Declining": "#ef4444", "Fading": "#94a3b8"
263
- }
264
- phase = result.get("phase", "Unknown")
265
- icon = phase_icons.get(phase, "โ“")
266
- color = phase_colors.get(phase, "#6b7280")
267
-
268
- phase_md = f"## {icon} {phase.upper()}"
269
-
270
- scores_md = f"""### ๐Ÿ“Š Metrics
271
- | | |
272
- |---|---|
273
- | **Confidence** | `{result.get('confidence', 'โ€”')}%` |
274
- | **Momentum Score** | `{result.get('momentum_score', 'โ€”')} / 100` |
275
- | **Media Spread** | `{result.get('media_spread', 'โ€”')}` |
276
- | **Articles Fetched** | `{len(articles)}` |
277
- | **Chunks in Vector Store** | `{len(chunks)}` |
278
- | **Chunks Retrieved via RAG** | `{len(retrieved_docs)}` |"""
279
-
280
- summary_md = f"### ๐Ÿ“ Narrative\n{result.get('summary', '')}"
281
-
282
- signals_md = "### ๐Ÿ” Key Signals\n" + "\n".join(
283
- [f"- {s}" for s in result.get('key_signals', [])]
284
- ) + "\n\n**Top Themes:** " + " ".join(
285
- [f"`{t}`" for t in result.get('top_themes', [])]
286
  )
287
 
 
 
 
 
 
 
 
288
  prediction_md = f"### ๐Ÿ”ฎ 30-Day Prediction\n_{result.get('prediction_30d', '')}_"
289
-
290
- sources_md = "### ๐Ÿ“ฐ Articles Used in RAG Retrieval\n" + "\n".join([
291
  f"- **{m['source']}** ({m['published_at']}): _{m['title']}_"
292
  for m in retrieved_meta[:6]
293
- ])
294
 
295
  return phase_md, scores_md, summary_md, signals_md, prediction_md, sources_md
296
 
297
  except json.JSONDecodeError:
298
- return ("โš ๏ธ Could not parse analysis response. Please try again.", *EMPTY[1:])
299
  except Exception as e:
300
- return (f"โš ๏ธ Error: {str(e)}", *EMPTY[1:])
 
 
 
 
 
 
 
301
 
302
 
303
  # โ”€โ”€โ”€ GRADIO UI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
@@ -315,24 +364,28 @@ footer { display: none !important; }
315
  """
316
 
317
  HOWTO = """
318
- **Step 1 โ€” Fetch:** NewsAPI retrieves up to 50 articles matching your topic from the selected date range.
319
 
320
- **Step 2 โ€” Chunk:** LangChain's `RecursiveCharacterTextSplitter` breaks each article into overlapping 400-character chunks (~80 words). Overlap prevents information loss at chunk boundaries.
321
 
322
- **Step 3 โ€” Embed:** Each chunk is encoded into a 384-dimensional dense vector using `all-MiniLM-L6-v2` (Sentence Transformers) โ€” optimised for semantic similarity, runs without a GPU.
323
 
324
- **Step 4 โ€” Store:** All vectors are loaded into a ChromaDB `EphemeralClient` collection (fully in-memory โ€” no disk writes, stateless per request).
325
 
326
- **Step 5 โ€” Retrieve:** A trend-signal query is run against the vector store. ChromaDB uses cosine similarity to return the 10 most semantically relevant chunks from across all articles.
327
 
328
- **Step 6 โ€” Generate:** Gemini Flash receives *only* the retrieved chunks as context โ€” not all 50 articles. This is the core RAG advantage: focused, grounded, efficient analysis with a minimal prompt. `response_mime_type="application/json"` guarantees clean structured output.
329
  """
330
 
331
- with gr.Blocks(theme=gr.themes.Soft(
332
- primary_hue="blue",
333
- secondary_hue="slate",
334
- font=[gr.themes.GoogleFont("DM Sans"), "ui-sans-serif", "sans-serif"]
335
- ), css=CSS, title="Trend Longevity Analyser") as demo:
 
 
 
 
336
 
337
  with gr.Column(elem_id="title-block"):
338
  gr.Markdown("""
@@ -342,39 +395,46 @@ with gr.Blocks(theme=gr.themes.Soft(
342
 
343
  gr.Markdown("---")
344
 
345
- # โ”€โ”€ Inputs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
346
  with gr.Accordion("๐Ÿ”‘ API Keys", open=False):
347
  gr.Markdown(
348
- "_Keys entered here are used only for your request and never stored. "
349
- "If you're hosting this Space privately, set `NEWSAPI_KEY` and "
350
- "`GEMINI_API_KEY` as Secrets instead._"
351
  )
352
  with gr.Row():
353
- newsapi_input = gr.Textbox(label="NewsAPI Key", placeholder="Get free key at newsapi.org", type="password")
354
- gemini_input = gr.Textbox(label="Google Gemini API Key", placeholder="Get free key at aistudio.google.com", type="password")
355
 
356
  with gr.Row():
357
  with gr.Column(scale=4):
358
  topic_input = gr.Textbox(
359
- label="Topic / Trend",
360
- placeholder="e.g. 'Gen Z workplace burnout' ยท 'green hydrogen' ยท 'AI companions'",
361
- lines=1
 
362
  )
363
  with gr.Column(scale=1):
364
- days_input = gr.Slider(label="Days Back", minimum=7, maximum=30, value=30, step=7,
365
- info="Free NewsAPI tier = 30 days max")
 
 
 
 
 
 
 
 
 
366
 
367
  analyse_btn = gr.Button("๐Ÿ” Analyse Trend", variant="primary", size="lg")
368
 
369
  gr.Markdown("---")
370
 
371
- # โ”€โ”€ Outputs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
372
  phase_output = gr.Markdown(elem_classes=["phase-box"])
373
 
374
  with gr.Row():
375
  with gr.Column():
376
- scores_output = gr.Markdown(elem_classes=["metric-card"])
377
- signals_output = gr.Markdown()
378
  with gr.Column():
379
  summary_output = gr.Markdown()
380
  prediction_output = gr.Markdown()
@@ -386,18 +446,16 @@ with gr.Blocks(theme=gr.themes.Soft(
386
 
387
  gr.Markdown(
388
  "_Built by [Sammie Wong](https://linkedin.com/in/) ยท "
389
- "Source: [GitHub](https://github.com/) ยท "
390
  "Powered by [NewsAPI](https://newsapi.org) + [ChromaDB](https://www.trychroma.com) + "
391
- "[Gemini Flash](https://aistudio.google.com)_",
392
  )
393
 
394
- # โ”€โ”€ Wire up โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
395
  analyse_btn.click(
396
  fn=run_analysis,
397
- inputs=[topic_input, days_input, newsapi_input, gemini_input],
398
  outputs=[phase_output, scores_output, summary_output,
399
- signals_output, prediction_output, sources_output]
400
  )
401
 
402
  if __name__ == "__main__":
403
- demo.launch()
 
9
  2. CHUNK โ†’ LangChain splits each article into overlapping 400-char chunks
10
  3. EMBED โ†’ Sentence Transformers encodes every chunk as a dense vector
11
  4. STORE โ†’ ChromaDB holds all vectors in an in-memory collection
12
+ 5. RETRIEVE โ†’ Semantic search returns the 6 most relevant chunks
13
  6. GENERATE โ†’ Gemini Flash analyses the retrieved context and returns a structured signal
14
 
15
  Built by: Sammie Wong
 
18
  import os
19
  import json
20
  import uuid
21
+ import time
22
  import gradio as gr
23
  import google.generativeai as genai
24
  import chromadb
25
  from chromadb.utils import embedding_functions
26
  from newsapi import NewsApiClient
27
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
28
  from datetime import datetime, timedelta
29
 
30
 
31
  # โ”€โ”€โ”€ STEP 1: FETCH โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
32
 
33
+ def fetch_articles(topic: str, days_back: int, newsapi_key: str,
34
+ boolean_query: str = "") -> list[dict]:
35
  """
36
+ Pull recent news articles from NewsAPI.
37
+
38
+ Two modes:
39
+ Boolean mode โ€” when boolean_query is provided, passed directly to NewsAPI
40
+ q parameter. Supports AND, OR, NOT, "exact phrase", (grouping).
41
+ No automatic quoting or fallback โ€” you control the query precisely.
42
+ Simple mode โ€” two-pass automatic:
43
+ Pass 1: exact phrase match e.g. "green hydrogen"
44
+ Pass 2: broad keyword fallback if < 5 results returned
45
+
46
+ Free tier: 100 requests/day. Boolean = 1 request; simple = 1โ€“2 requests.
47
  """
48
  client = NewsApiClient(api_key=newsapi_key)
49
  from_date = (datetime.now() - timedelta(days=days_back)).strftime('%Y-%m-%d')
50
 
51
+ common_params = dict(
 
52
  language='en',
53
  sort_by='publishedAt',
54
  from_param=from_date,
55
+ page_size=50,
56
  )
57
+
58
+ if boolean_query.strip():
59
+ # Boolean mode: pass query directly, no modification
60
+ response = client.get_everything(q=boolean_query.strip(), **common_params)
61
+ return response.get('articles', [])
62
+
63
+ # Simple mode: two-pass
64
+ response = client.get_everything(q=f'"{topic}"', **common_params)
65
+ articles = response.get('articles', [])
66
+ if len(articles) < 5:
67
+ response = client.get_everything(q=topic, **common_params)
68
+ articles = response.get('articles', [])
69
+ return articles
70
 
71
 
72
  # โ”€โ”€โ”€ STEP 2: CHUNK โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
80
  losing meaning at chunk boundaries.
81
  """
82
  splitter = RecursiveCharacterTextSplitter(
83
+ chunk_size=400,
84
+ chunk_overlap=40,
85
  )
86
 
87
  all_chunks, all_metadata = [], []
88
 
89
  for article in articles:
 
90
  full_text = " ".join(filter(None, [
91
  article.get('title', ''),
92
  article.get('description', ''),
93
+ article.get('content', ''),
94
  ]))
95
 
96
  for chunk in splitter.split_text(full_text):
 
98
  all_metadata.append({
99
  'source': article['source']['name'],
100
  'published_at': article['publishedAt'][:10],
101
+ 'title': (article.get('title') or '')[:100],
102
  })
103
 
104
  return all_chunks, all_metadata
 
112
  in an in-memory ChromaDB collection.
113
 
114
  Model: all-MiniLM-L6-v2 โ€” fast, accurate, no GPU needed.
115
+ EphemeralClient = fully in-memory, stateless per request.
 
116
  """
117
  embedding_fn = embedding_functions.SentenceTransformerEmbeddingFunction(
118
  model_name="all-MiniLM-L6-v2"
119
  )
120
 
121
  try:
122
+ db = chromadb.EphemeralClient()
123
  except AttributeError:
124
+ db = chromadb.Client()
125
 
126
  collection = db.create_collection(
127
  name=f"trend_{uuid.uuid4().hex[:8]}",
128
+ embedding_function=embedding_fn,
129
  )
130
 
 
131
  batch_size = 100
132
  for i in range(0, len(chunks), batch_size):
133
+ batch = chunks[i:i + batch_size]
134
  collection.add(
135
+ documents=batch,
136
+ metadatas=metadatas[i:i + batch_size],
137
+ ids=[f"c{i + j}" for j in range(len(batch))],
138
  )
139
 
140
  return collection
 
142
 
143
  # โ”€โ”€โ”€ STEP 5: RETRIEVE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
144
 
145
+ def retrieve_context(collection, topic: str, n_results: int = 6) -> tuple[list[str], list[dict]]:
146
  """
147
+ Retrieve the most relevant chunks via cosine similarity.
148
 
149
+ n_results=6 (down from 10) keeps the Gemini prompt comfortably
150
+ under the free-tier token-per-minute limit while preserving signal quality.
151
  """
152
  query = f"trend momentum growth decline viral popularity media coverage {topic}"
153
 
154
  results = collection.query(
155
  query_texts=[query],
156
+ n_results=min(n_results, collection.count()),
157
  )
158
  return results['documents'][0], results['metadatas'][0]
159
 
160
 
161
  # โ”€โ”€โ”€ STEP 6: GENERATE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
162
 
163
+ # Hard cap on context chars sent to Gemini.
164
+ # 6 chunks ร— 400 chars = ~2400 chars raw. We cap at 3000 to stay well
165
+ # under the free-tier 32K tokens-per-minute limit.
166
+ MAX_CONTEXT_CHARS = 3000
167
+
168
  def analyse_with_gemini(
169
  topic: str,
170
  retrieved_docs: list[str],
171
  retrieved_meta: list[dict],
172
  total_articles: int,
173
  days_back: int,
174
+ gemini_key: str,
175
  ) -> dict:
176
  """
177
+ Pass retrieved chunks to Gemini Flash for trend analysis.
 
 
 
 
178
 
179
+ Reliability features:
180
+ - Context is hard-capped at MAX_CONTEXT_CHARS to prevent 429 token-rate errors.
181
+ - Auto-retry: on a 429 response, waits 15 s then retries up to 2 more times.
182
+ - response_mime_type="application/json" forces clean JSON โ€” no parsing issues.
183
  """
184
+ # Build context, truncating to stay inside token-rate limit
185
+ context_parts = []
186
+ total_chars = 0
187
+ for doc, m in zip(retrieved_docs, retrieved_meta):
188
+ entry = f"[{m['source']} ยท {m['published_at']}]\n{doc}"
189
+ if total_chars + len(entry) > MAX_CONTEXT_CHARS:
190
+ # Include a truncated version rather than skipping entirely
191
+ remaining = MAX_CONTEXT_CHARS - total_chars
192
+ if remaining > 80:
193
+ context_parts.append(entry[:remaining] + "โ€ฆ")
194
+ break
195
+ context_parts.append(entry)
196
+ total_chars += len(entry)
197
+
198
+ context = "\n\n".join(context_parts)
199
 
200
  prompt = f"""You are a senior trend analyst at a social listening company (like Pulsar, Brandwatch, or Sprinklr).
201
 
202
+ You have been given the {len(context_parts)} most semantically relevant news excerpts about "{topic}",
203
  retrieved via RAG from a corpus of {total_articles} articles published in the last {days_back} days.
204
 
205
  RETRIEVED CONTEXT:
 
233
  model = genai.GenerativeModel(
234
  model_name="gemini-2.0-flash",
235
  generation_config=genai.GenerationConfig(
236
+ response_mime_type="application/json",
237
+ max_output_tokens=600,
238
+ ),
239
  )
240
+
241
+ # Retry loop: up to 3 attempts, 15-second pause on 429
242
+ last_error = None
243
+ for attempt in range(3):
244
+ try:
245
+ response = model.generate_content(prompt)
246
+ return json.loads(response.text)
247
+ except Exception as e:
248
+ last_error = e
249
+ if "429" in str(e) and attempt < 2:
250
+ wait = 15 * (attempt + 1) # 15 s, then 30 s
251
+ time.sleep(wait)
252
+ continue
253
+ break
254
+
255
+ raise last_error
256
 
257
 
258
  # โ”€โ”€โ”€ PIPELINE ORCHESTRATOR โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
259
 
260
+ def run_analysis(topic: str, days_back: int, newsapi_key_input: str, gemini_key_input: str,
261
+ boolean_query: str = ""):
262
  """
263
+ Orchestrates the full 6-step RAG pipeline.
264
+ Returns 6 Markdown strings for the Gradio output components.
265
  """
266
  EMPTY = ("", "", "", "", "", "")
267
 
268
  if not topic.strip():
269
+ return ("โš ๏ธ Please enter a topic label (used for the Gemini prompt even in Boolean mode).", *EMPTY[1:])
270
 
 
271
  news_key = newsapi_key_input.strip() or os.environ.get("NEWSAPI_KEY", "")
272
  gemini_key = gemini_key_input.strip() or os.environ.get("GEMINI_API_KEY", "")
273
 
274
  if not news_key:
275
+ return ("โš ๏ธ NewsAPI key required. Free key at newsapi.org", *EMPTY[1:])
276
  if not gemini_key:
277
+ return ("โš ๏ธ Gemini key required. Free key at aistudio.google.com", *EMPTY[1:])
278
+
279
+ using_boolean = bool(boolean_query.strip())
280
+ search_label = boolean_query.strip() if using_boolean else topic
281
 
282
  try:
283
+ # Step 1
284
+ articles = fetch_articles(topic, int(days_back), news_key, boolean_query)
285
  if not articles:
286
+ hint = (
287
+ f"Boolean query returned no results: `{boolean_query}`\n\nCheck your operators and try broader terms."
288
+ if using_boolean else
289
+ f"No articles found for **'{topic}'** in the last {days_back} days.\n\nTry a shorter or broader topic name."
290
  )
291
+ return (f"โš ๏ธ {hint}", *EMPTY[1:])
292
 
293
+ # Steps 2โ€“4
294
  chunks, metadatas = chunk_articles(articles)
295
  collection = build_vector_store(chunks, metadatas)
296
 
297
+ # Step 5
298
  retrieved_docs, retrieved_meta = retrieve_context(collection, topic)
299
 
300
+ # Step 6
301
  result = analyse_with_gemini(
302
  topic, retrieved_docs, retrieved_meta,
303
+ len(articles), int(days_back), gemini_key,
304
  )
305
 
306
+ # โ”€โ”€ Format outputs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
307
+ phase_icons = {"Early": "๐ŸŒฑ", "Rising": "๐Ÿ“ˆ", "Peak": "๐Ÿ”ฅ", "Declining": "๐Ÿ“‰", "Fading": "๐ŸŒซ๏ธ"}
308
+ phase = result.get("phase", "Unknown")
309
+ icon = phase_icons.get(phase, "โ“")
310
+ phase_md = f"## {icon} {phase.upper()}"
311
+
312
+ mode_label = f"Boolean: `{boolean_query.strip()}`" if using_boolean else "Simple (auto)"
313
+ scores_md = (
314
+ f"### ๐Ÿ“Š Metrics\n| | |\n|---|---|\n"
315
+ f"| **Confidence** | `{result.get('confidence', 'โ€”')}%` |\n"
316
+ f"| **Momentum Score** | `{result.get('momentum_score', 'โ€”')} / 100` |\n"
317
+ f"| **Media Spread** | `{result.get('media_spread', 'โ€”')}` |\n"
318
+ f"| **Search Mode** | {mode_label} |\n"
319
+ f"| **Articles Fetched** | `{len(articles)}` |\n"
320
+ f"| **Chunks in Vector Store** | `{len(chunks)}` |\n"
321
+ f"| **Chunks Sent to Gemini** | `{len(retrieved_docs)}` |"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
322
  )
323
 
324
+ summary_md = f"### ๐Ÿ“ Narrative\n{result.get('summary', '')}"
325
+ signals_md = (
326
+ "### ๐Ÿ” Key Signals\n"
327
+ + "\n".join(f"- {s}" for s in result.get('key_signals', []))
328
+ + "\n\n**Top Themes:** "
329
+ + " ".join(f"`{t}`" for t in result.get('top_themes', []))
330
+ )
331
  prediction_md = f"### ๐Ÿ”ฎ 30-Day Prediction\n_{result.get('prediction_30d', '')}_"
332
+ sources_md = "### ๐Ÿ“ฐ Sources Retrieved via RAG\n" + "\n".join(
 
333
  f"- **{m['source']}** ({m['published_at']}): _{m['title']}_"
334
  for m in retrieved_meta[:6]
335
+ )
336
 
337
  return phase_md, scores_md, summary_md, signals_md, prediction_md, sources_md
338
 
339
  except json.JSONDecodeError:
340
+ return ("โš ๏ธ Could not parse Gemini response. Please try again.", *EMPTY[1:])
341
  except Exception as e:
342
+ err = str(e)
343
+ if "429" in err:
344
+ return (
345
+ "โš ๏ธ **Rate limit hit (429).** Gemini free tier allows ~15 requests/minute.\n\n"
346
+ "Wait 60 seconds and try again โ€” the retry logic above handled 2 attempts already.",
347
+ *EMPTY[1:]
348
+ )
349
+ return (f"โš ๏ธ Error: {err}", *EMPTY[1:])
350
 
351
 
352
  # โ”€โ”€โ”€ GRADIO UI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
364
  """
365
 
366
  HOWTO = """
367
+ **Step 1 โ€” Fetch:** NewsAPI retrieves up to 50 articles. Two modes: *Simple* (auto exact-phrase then broad fallback) or *Boolean* โ€” where your query is passed directly to NewsAPI supporting `AND`, `OR`, `NOT`, `"exact phrase"`, and `(grouping)`. Boolean mode gives you precise control over what enters the RAG corpus.
368
 
369
+ **Step 2 โ€” Chunk:** LangChain's `RecursiveCharacterTextSplitter` breaks each article into overlapping 400-character chunks. Overlap prevents information loss at chunk boundaries.
370
 
371
+ **Step 3 โ€” Embed:** Each chunk is encoded into a 384-dimensional dense vector using `all-MiniLM-L6-v2` (Sentence Transformers) โ€” no GPU needed.
372
 
373
+ **Step 4 โ€” Store:** All vectors are loaded into a ChromaDB `EphemeralClient` collection (fully in-memory, stateless per request).
374
 
375
+ **Step 5 โ€” Retrieve:** A trend-signal query runs against the vector store. ChromaDB returns the 6 most semantically relevant chunks via cosine similarity.
376
 
377
+ **Step 6 โ€” Generate:** Gemini Flash receives only the retrieved chunks (capped at 3,000 chars) โ€” not all 50 articles. This keeps the prompt inside the free-tier token-rate limit and is the core RAG advantage. Auto-retries on 429 with a 15-second wait.
378
  """
379
 
380
+ with gr.Blocks(
381
+ theme=gr.themes.Soft(
382
+ primary_hue="blue",
383
+ secondary_hue="slate",
384
+ font=[gr.themes.GoogleFont("DM Sans"), "ui-sans-serif", "sans-serif"],
385
+ ),
386
+ css=CSS,
387
+ title="Trend Longevity Analyser",
388
+ ) as demo:
389
 
390
  with gr.Column(elem_id="title-block"):
391
  gr.Markdown("""
 
395
 
396
  gr.Markdown("---")
397
 
 
398
  with gr.Accordion("๐Ÿ”‘ API Keys", open=False):
399
  gr.Markdown(
400
+ "_Keys are used only for your request and never stored. "
401
+ "Set `NEWSAPI_KEY` and `GEMINI_API_KEY` as Space Secrets to avoid entering them each time._"
 
402
  )
403
  with gr.Row():
404
+ newsapi_input = gr.Textbox(label="NewsAPI Key", placeholder="newsapi.org โ€” free", type="password")
405
+ gemini_input = gr.Textbox(label="Google Gemini API Key", placeholder="aistudio.google.com โ€” free", type="password")
406
 
407
  with gr.Row():
408
  with gr.Column(scale=4):
409
  topic_input = gr.Textbox(
410
+ label="Topic Label",
411
+ placeholder="e.g. Gen Z workplace burnout ยท green hydrogen ยท AI companions",
412
+ info="Used as the display name and passed to Gemini. Always required.",
413
+ lines=1,
414
  )
415
  with gr.Column(scale=1):
416
+ days_input = gr.Slider(
417
+ label="Days Back", minimum=7, maximum=30, value=30, step=7,
418
+ info="Free NewsAPI tier = 30 days max",
419
+ )
420
+
421
+ boolean_input = gr.Textbox(
422
+ label="โšก Boolean Search Query (optional โ€” overrides simple search)",
423
+ placeholder='e.g. "Gen Z" AND (burnout OR "quiet quitting") NOT productivity',
424
+ info='Operators: AND OR NOT "exact phrase" (grouping) โ€” passed directly to NewsAPI',
425
+ lines=1,
426
+ )
427
 
428
  analyse_btn = gr.Button("๐Ÿ” Analyse Trend", variant="primary", size="lg")
429
 
430
  gr.Markdown("---")
431
 
 
432
  phase_output = gr.Markdown(elem_classes=["phase-box"])
433
 
434
  with gr.Row():
435
  with gr.Column():
436
+ scores_output = gr.Markdown(elem_classes=["metric-card"])
437
+ signals_output = gr.Markdown()
438
  with gr.Column():
439
  summary_output = gr.Markdown()
440
  prediction_output = gr.Markdown()
 
446
 
447
  gr.Markdown(
448
  "_Built by [Sammie Wong](https://linkedin.com/in/) ยท "
 
449
  "Powered by [NewsAPI](https://newsapi.org) + [ChromaDB](https://www.trychroma.com) + "
450
+ "[Gemini Flash](https://aistudio.google.com)_"
451
  )
452
 
 
453
  analyse_btn.click(
454
  fn=run_analysis,
455
+ inputs=[topic_input, days_input, newsapi_input, gemini_input, boolean_input],
456
  outputs=[phase_output, scores_output, summary_output,
457
+ signals_output, prediction_output, sources_output],
458
  )
459
 
460
  if __name__ == "__main__":
461
+ demo.launch()