plexdx commited on
Commit
1e26db6
Β·
verified Β·
1 Parent(s): c5175d5

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +43 -87
app.py CHANGED
@@ -3,6 +3,7 @@ import numpy as np
3
  import warnings
4
  import feedparser
5
  from datetime import datetime
 
6
 
7
  warnings.filterwarnings('ignore')
8
  print("βœ… Core imports done.")
@@ -116,7 +117,6 @@ if not headlines:
116
  news_df = pd.DataFrame(headlines)
117
  news_df['published_at'] = pd.to_datetime(news_df['published_at'], errors='coerce', utc=True)
118
  print(f"βœ… Live news loaded: {len(news_df)} headlines from {news_df['source'].nunique()} sources")
119
- news_df.head(3)
120
 
121
  from transformers import pipeline
122
  from sentence_transformers import SentenceTransformer
@@ -195,7 +195,6 @@ INVENTED_INSTITUTIONS = re.compile(
195
  )
196
 
197
  def get_sentiment_score(text: str) -> float:
198
- """Returns float in [-1, 1]. Negative = negative sentiment."""
199
  try:
200
  result = sentiment_pipeline(text[:512])[0]
201
  score = result['score']
@@ -204,29 +203,22 @@ def get_sentiment_score(text: str) -> float:
204
  return 0.0
205
 
206
  def get_source_credibility(source: str) -> float:
207
- """Lookup against known domain credibility scores."""
208
  for domain, score in SOURCE_CREDIBILITY.items():
209
  if domain.lower() in source.lower():
210
  return score
211
- return 0.5 # unknown source β†’ uncertain
212
 
213
  def get_citation_anomaly_score(text: str) -> float:
214
- """Detects patterns common in hallucinated citations."""
215
  score = 0.0
216
- # Fake DOI pattern
217
  if FAKE_DOI_PATTERN.search(text): score += 0.3
218
- # Impossible year references
219
  if IMPOSSIBLE_YEAR.search(text): score += 0.3
220
- # Suspicious institution names
221
  if INVENTED_INSTITUTIONS.search(text): score += 0.4
222
  return min(score, 1.0)
223
 
224
  def get_semantic_similarity(text: str, k: int = 3) -> float:
225
- """Cosine similarity of input against top-k trusted FAISS facts."""
226
  try:
227
  emb = embedder.encode([text], convert_to_numpy=True).astype(np.float32)
228
  distances, _ = faiss_index.search(emb, k)
229
- # Convert L2 distance to similarity (lower distance = higher similarity)
230
  avg_dist = np.mean(distances[0])
231
  similarity = 1.0 / (1.0 + avg_dist)
232
  return float(np.clip(similarity, 0, 1))
@@ -234,21 +226,18 @@ def get_semantic_similarity(text: str, k: int = 3) -> float:
234
  return 0.5
235
 
236
  def get_nli_contradiction_score(claim: str, references: list) -> float:
237
- """DeBERTa NLI: fraction of references that contradict the claim."""
238
  try:
239
  result = nli_pipeline(
240
  claim,
241
  candidate_labels=["entailment", "neutral", "contradiction"],
242
  hypothesis_template="This claim is related to: {}",
243
  )
244
- # Get contradiction score
245
  scores = dict(zip(result['labels'], result['scores']))
246
  return float(scores.get('contradiction', 0.0))
247
  except:
248
  return 0.5
249
 
250
  def retrieve_reference_sentences(claim: str, k: int = 5) -> list:
251
- """Retrieve top-k relevant facts from FAISS index."""
252
  try:
253
  emb = embedder.encode([claim], convert_to_numpy=True).astype(np.float32)
254
  _, indices = faiss_index.search(emb, k)
@@ -258,40 +247,11 @@ def retrieve_reference_sentences(claim: str, k: int = 5) -> list:
258
 
259
  print("βœ… Feature extraction functions defined.")
260
 
261
- # ── Compute Features on a Sample ──────────────────────────────────────────────
262
- SAMPLE_TEXTS = [
263
- "The moon is made of cheese.",
264
- "Water boils at 100Β°C at sea level.",
265
- "Scientists discovered that 5G towers emit mind-control frequencies.",
266
- "The Eiffel Tower is 330 meters tall.",
267
- "According to a 2031 study from the Institute of Neural Enhancement, humans only use 10% of their brain.",
268
- ]
269
-
270
- rows = []
271
- for text in SAMPLE_TEXTS:
272
- refs = retrieve_reference_sentences(text)
273
- row = {
274
- 'text': text[:60] + '...' if len(text) > 60 else text,
275
- 'sentiment_score': get_sentiment_score(text),
276
- 'source_credibility': 0.5, # unknown source for these samples
277
- 'nli_contradiction_score': get_nli_contradiction_score(text, refs),
278
- 'citation_anomaly_score': get_citation_anomaly_score(text),
279
- 'semantic_similarity': get_semantic_similarity(text),
280
- }
281
- rows.append(row)
282
-
283
- features_df = pd.DataFrame(rows)
284
- print("βœ… Feature matrix computed:")
285
- features_df
286
-
287
  # ── A. Fake News Classifier (LIAR β†’ 3-class) ──────────────────────────────────
288
  from sklearn.linear_model import LogisticRegression
289
- from sklearn.preprocessing import LabelEncoder
290
  from sklearn.model_selection import train_test_split
291
  from sklearn.metrics import classification_report
292
- import numpy as np
293
 
294
- # Collapse LIAR 6-class to 3-class
295
  LIAR_MAP = {
296
  'pants-fire': 'misinformation',
297
  'false': 'misinformation',
@@ -304,37 +264,27 @@ LIAR_MAP = {
304
  liar_sample = liar_df.sample(min(500, len(liar_df)), random_state=42).copy()
305
  liar_sample['label_3'] = liar_sample['label'].map(LIAR_MAP).fillna('uncertain')
306
 
307
- # Encode statements β†’ embeddings for classifier
308
  print("Encoding LIAR statements...")
309
- X_liar = embedder.encode(liar_sample['statement'].tolist(), show_progress_bar=True)
310
  y_liar = liar_sample['label_3'].values
311
 
312
  X_train, X_test, y_train, y_test = train_test_split(X_liar, y_liar, test_size=0.2, random_state=42)
313
 
314
  fake_news_clf = LogisticRegression(max_iter=500, random_state=42)
315
  fake_news_clf.fit(X_train, y_train)
316
-
317
- print("\nπŸ“Š Fake News Classifier Report:")
318
- print(classification_report(y_test, fake_news_clf.predict(X_test)))
319
  print("βœ… Fake news classifier trained.")
320
 
321
  # ── B. Hallucination Scorer ───────────────────────────────────────────────────
322
-
323
  def score_hallucination(claim: str) -> dict:
324
- """
325
- Scores a single claim for hallucination risk.
326
- Returns dict with hallucination_risk [0-100] and evidence snippets.
327
- """
328
  try:
329
  references = retrieve_reference_sentences(claim, k=5)
330
  contradiction_score = get_nli_contradiction_score(claim, references)
331
  similarity = get_semantic_similarity(claim)
332
  citation_anomaly = get_citation_anomaly_score(claim)
333
 
334
- # Weighted combination
335
  raw_risk = (
336
  0.50 * contradiction_score +
337
- 0.30 * (1 - similarity) + # low similarity to trusted facts = higher risk
338
  0.20 * citation_anomaly
339
  )
340
  hallucination_risk = int(np.clip(raw_risk * 100, 0, 100))
@@ -349,14 +299,6 @@ def score_hallucination(claim: str) -> dict:
349
  return {'hallucination_risk': 50, 'contradiction_score': 0.5,
350
  'semantic_similarity': 0.5, 'evidence_snippets': [], 'error': str(e)}
351
 
352
- # Test
353
- test_claims = [
354
- "The moon is made of cheese.",
355
- "Water boils at 100 degrees Celsius at sea level.",
356
- ]
357
- for claim in test_claims:
358
- result = score_hallucination(claim)
359
- print(f" '{claim[:50]}...' β†’ risk: {result['hallucination_risk']}%")
360
  print("βœ… Hallucination scorer working.")
361
 
362
  # ── C. Event Volatility Forecaster ───────────────────────────────────────────
@@ -369,14 +311,12 @@ except ImportError:
369
  print("⚠️ statsforecast not available, using EWMA fallback.")
370
 
371
  def compute_volatility_series(df: pd.DataFrame, window: int = 7) -> pd.Series:
372
- """Rolling std of sentiment scores over headlines."""
373
  df = df.copy().sort_values('published_at')
374
  sentiments = df['headline'].apply(get_sentiment_score)
375
  volatility = sentiments.rolling(window=min(window, len(df)), min_periods=1).std().fillna(0)
376
  return volatility
377
 
378
  def forecast_volatility(series: pd.Series, horizon: int = 3) -> dict:
379
- """Forecast next `horizon` periods of volatility."""
380
  if HAS_STATSFORECAST and len(series) >= 10:
381
  try:
382
  sf_df = pd.DataFrame({
@@ -392,33 +332,26 @@ def forecast_volatility(series: pd.Series, horizon: int = 3) -> dict:
392
  except:
393
  pass
394
 
395
- # EWMA fallback
396
  ewma = series.ewm(span=min(5, len(series))).mean()
397
  last = ewma.iloc[-1]
398
  forecasted = [last * (1 + 0.02 * i) for i in range(1, horizon + 1)]
399
  trend = 'rising' if forecasted[-1] > series.mean() else 'stable'
400
  return {'method': 'EWMA', 'forecast': forecasted, 'trend': trend}
401
 
402
- volatility_series = compute_volatility_series(news_df)
403
- forecast_result = forecast_volatility(volatility_series)
404
- print(f"βœ… Volatility forecast: {forecast_result['method']} β†’ trend: {forecast_result['trend']}")
405
-
406
  # ── D. Final Risk Score Aggregator ────────────────────────────────────────────
407
- # Configurable weights (adjust these constants)
408
  W_HALLUCINATION = 0.40
409
  W_FAKE_NEWS = 0.35
410
  W_CITATION = 0.15
411
  W_SIMILARITY = 0.10
412
 
413
  COLOR_MAP = {
414
- 'confirmed': 'rgba(52, 199, 89, 0.15)', # green
415
- 'uncertain': 'rgba(255, 204, 0, 0.15)', # yellow
416
- 'misinformation':'rgba(255, 59, 48, 0.15)', # red
417
- 'hallucination': 'rgba(175, 82, 222, 0.15)', # purple
418
  }
419
 
420
  def get_fake_news_probability(text: str) -> tuple[str, float]:
421
- """Returns (label, probability) from fake news classifier."""
422
  try:
423
  emb = embedder.encode([text])
424
  proba = fake_news_clf.predict_proba(emb)[0]
@@ -430,22 +363,15 @@ def get_fake_news_probability(text: str) -> tuple[str, float]:
430
  return 'uncertain', 0.5
431
 
432
  def analyze_text(text: str, source: str = 'unknown') -> dict:
433
- """
434
- Full pipeline: text β†’ JSON risk payload.
435
- This is the function the Gradio API exposes.
436
- """
437
  try:
438
- # --- feature extraction ---
439
  halu_result = score_hallucination(text)
440
  fake_label, fake_conf = get_fake_news_probability(text)
441
  citation_score = get_citation_anomaly_score(text)
442
  similarity = get_semantic_similarity(text)
443
  credibility = get_source_credibility(source)
444
 
445
- # Normalise fake news label to a risk score
446
  fake_risk = {'misinformation': 0.9, 'uncertain': 0.5, 'credible': 0.1}.get(fake_label, 0.5)
447
 
448
- # Aggregate
449
  combined_risk = (
450
  W_HALLUCINATION * (halu_result['hallucination_risk'] / 100) +
451
  W_FAKE_NEWS * fake_risk +
@@ -454,7 +380,6 @@ def analyze_text(text: str, source: str = 'unknown') -> dict:
454
  )
455
  combined_risk = float(np.clip(combined_risk, 0, 1))
456
 
457
- # Determine status
458
  if combined_risk < 0.25:
459
  status = 'confirmed'
460
  elif combined_risk < 0.55:
@@ -464,7 +389,7 @@ def analyze_text(text: str, source: str = 'unknown') -> dict:
464
  else:
465
  status = 'misinformation'
466
 
467
- confidence = abs(combined_risk - 0.5) * 2 # distance from uncertain midpoint
468
 
469
  tooltip = (
470
  f"{status.title()} risk: {int(combined_risk*100)}%. "
@@ -493,6 +418,37 @@ def analyze_text(text: str, source: str = 'unknown') -> dict:
493
  'evidence_snippets': []
494
  }
495
 
496
- # Quick smoke test
497
- test = analyze_text("The moon is made of cheese.")
498
- print(f"βœ… Aggregator test: status={test['status']}, risk={test['combined_risk']}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  import warnings
4
  import feedparser
5
  from datetime import datetime
6
+ import gradio as gr
7
 
8
  warnings.filterwarnings('ignore')
9
  print("βœ… Core imports done.")
 
117
  news_df = pd.DataFrame(headlines)
118
  news_df['published_at'] = pd.to_datetime(news_df['published_at'], errors='coerce', utc=True)
119
  print(f"βœ… Live news loaded: {len(news_df)} headlines from {news_df['source'].nunique()} sources")
 
120
 
121
  from transformers import pipeline
122
  from sentence_transformers import SentenceTransformer
 
195
  )
196
 
197
  def get_sentiment_score(text: str) -> float:
 
198
  try:
199
  result = sentiment_pipeline(text[:512])[0]
200
  score = result['score']
 
203
  return 0.0
204
 
205
  def get_source_credibility(source: str) -> float:
 
206
  for domain, score in SOURCE_CREDIBILITY.items():
207
  if domain.lower() in source.lower():
208
  return score
209
+ return 0.5
210
 
211
  def get_citation_anomaly_score(text: str) -> float:
 
212
  score = 0.0
 
213
  if FAKE_DOI_PATTERN.search(text): score += 0.3
 
214
  if IMPOSSIBLE_YEAR.search(text): score += 0.3
 
215
  if INVENTED_INSTITUTIONS.search(text): score += 0.4
216
  return min(score, 1.0)
217
 
218
  def get_semantic_similarity(text: str, k: int = 3) -> float:
 
219
  try:
220
  emb = embedder.encode([text], convert_to_numpy=True).astype(np.float32)
221
  distances, _ = faiss_index.search(emb, k)
 
222
  avg_dist = np.mean(distances[0])
223
  similarity = 1.0 / (1.0 + avg_dist)
224
  return float(np.clip(similarity, 0, 1))
 
226
  return 0.5
227
 
228
  def get_nli_contradiction_score(claim: str, references: list) -> float:
 
229
  try:
230
  result = nli_pipeline(
231
  claim,
232
  candidate_labels=["entailment", "neutral", "contradiction"],
233
  hypothesis_template="This claim is related to: {}",
234
  )
 
235
  scores = dict(zip(result['labels'], result['scores']))
236
  return float(scores.get('contradiction', 0.0))
237
  except:
238
  return 0.5
239
 
240
  def retrieve_reference_sentences(claim: str, k: int = 5) -> list:
 
241
  try:
242
  emb = embedder.encode([claim], convert_to_numpy=True).astype(np.float32)
243
  _, indices = faiss_index.search(emb, k)
 
247
 
248
  print("βœ… Feature extraction functions defined.")
249
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
250
  # ── A. Fake News Classifier (LIAR β†’ 3-class) ──────────────────────────────────
251
  from sklearn.linear_model import LogisticRegression
 
252
  from sklearn.model_selection import train_test_split
253
  from sklearn.metrics import classification_report
 
254
 
 
255
  LIAR_MAP = {
256
  'pants-fire': 'misinformation',
257
  'false': 'misinformation',
 
264
  liar_sample = liar_df.sample(min(500, len(liar_df)), random_state=42).copy()
265
  liar_sample['label_3'] = liar_sample['label'].map(LIAR_MAP).fillna('uncertain')
266
 
 
267
  print("Encoding LIAR statements...")
268
+ X_liar = embedder.encode(liar_sample['statement'].tolist(), show_progress_bar=False)
269
  y_liar = liar_sample['label_3'].values
270
 
271
  X_train, X_test, y_train, y_test = train_test_split(X_liar, y_liar, test_size=0.2, random_state=42)
272
 
273
  fake_news_clf = LogisticRegression(max_iter=500, random_state=42)
274
  fake_news_clf.fit(X_train, y_train)
 
 
 
275
  print("βœ… Fake news classifier trained.")
276
 
277
  # ── B. Hallucination Scorer ───────────────────────────────────────────────────
 
278
  def score_hallucination(claim: str) -> dict:
 
 
 
 
279
  try:
280
  references = retrieve_reference_sentences(claim, k=5)
281
  contradiction_score = get_nli_contradiction_score(claim, references)
282
  similarity = get_semantic_similarity(claim)
283
  citation_anomaly = get_citation_anomaly_score(claim)
284
 
 
285
  raw_risk = (
286
  0.50 * contradiction_score +
287
+ 0.30 * (1 - similarity) +
288
  0.20 * citation_anomaly
289
  )
290
  hallucination_risk = int(np.clip(raw_risk * 100, 0, 100))
 
299
  return {'hallucination_risk': 50, 'contradiction_score': 0.5,
300
  'semantic_similarity': 0.5, 'evidence_snippets': [], 'error': str(e)}
301
 
 
 
 
 
 
 
 
 
302
  print("βœ… Hallucination scorer working.")
303
 
304
  # ── C. Event Volatility Forecaster ───────────────────────────────────────────
 
311
  print("⚠️ statsforecast not available, using EWMA fallback.")
312
 
313
  def compute_volatility_series(df: pd.DataFrame, window: int = 7) -> pd.Series:
 
314
  df = df.copy().sort_values('published_at')
315
  sentiments = df['headline'].apply(get_sentiment_score)
316
  volatility = sentiments.rolling(window=min(window, len(df)), min_periods=1).std().fillna(0)
317
  return volatility
318
 
319
  def forecast_volatility(series: pd.Series, horizon: int = 3) -> dict:
 
320
  if HAS_STATSFORECAST and len(series) >= 10:
321
  try:
322
  sf_df = pd.DataFrame({
 
332
  except:
333
  pass
334
 
 
335
  ewma = series.ewm(span=min(5, len(series))).mean()
336
  last = ewma.iloc[-1]
337
  forecasted = [last * (1 + 0.02 * i) for i in range(1, horizon + 1)]
338
  trend = 'rising' if forecasted[-1] > series.mean() else 'stable'
339
  return {'method': 'EWMA', 'forecast': forecasted, 'trend': trend}
340
 
 
 
 
 
341
  # ── D. Final Risk Score Aggregator ────────────────────────────────────────────
 
342
  W_HALLUCINATION = 0.40
343
  W_FAKE_NEWS = 0.35
344
  W_CITATION = 0.15
345
  W_SIMILARITY = 0.10
346
 
347
  COLOR_MAP = {
348
+ 'confirmed': 'rgba(52, 199, 89, 0.15)',
349
+ 'uncertain': 'rgba(255, 204, 0, 0.15)',
350
+ 'misinformation':'rgba(255, 59, 48, 0.15)',
351
+ 'hallucination': 'rgba(175, 82, 222, 0.15)',
352
  }
353
 
354
  def get_fake_news_probability(text: str) -> tuple[str, float]:
 
355
  try:
356
  emb = embedder.encode([text])
357
  proba = fake_news_clf.predict_proba(emb)[0]
 
363
  return 'uncertain', 0.5
364
 
365
  def analyze_text(text: str, source: str = 'unknown') -> dict:
 
 
 
 
366
  try:
 
367
  halu_result = score_hallucination(text)
368
  fake_label, fake_conf = get_fake_news_probability(text)
369
  citation_score = get_citation_anomaly_score(text)
370
  similarity = get_semantic_similarity(text)
371
  credibility = get_source_credibility(source)
372
 
 
373
  fake_risk = {'misinformation': 0.9, 'uncertain': 0.5, 'credible': 0.1}.get(fake_label, 0.5)
374
 
 
375
  combined_risk = (
376
  W_HALLUCINATION * (halu_result['hallucination_risk'] / 100) +
377
  W_FAKE_NEWS * fake_risk +
 
380
  )
381
  combined_risk = float(np.clip(combined_risk, 0, 1))
382
 
 
383
  if combined_risk < 0.25:
384
  status = 'confirmed'
385
  elif combined_risk < 0.55:
 
389
  else:
390
  status = 'misinformation'
391
 
392
+ confidence = abs(combined_risk - 0.5) * 2
393
 
394
  tooltip = (
395
  f"{status.title()} risk: {int(combined_risk*100)}%. "
 
418
  'evidence_snippets': []
419
  }
420
 
421
+ # ── E. Main Web Application with Gradio ───────────────────────────────────────
422
+ def predict(text):
423
+ return analyze_text(text)
424
+
425
+ with gr.Blocks(theme=gr.themes.Soft()) as demo:
426
+ gr.Markdown("# πŸ›‘οΈ AI Risk & Fact-Checking Dashboard")
427
+ gr.Markdown("Analyze text for hallucination risk, fake news probability, and citation anomalies.")
428
+
429
+ with gr.Row():
430
+ with gr.Column():
431
+ input_text = gr.Textbox(
432
+ lines=5,
433
+ placeholder="Enter a claim or news snippet here...",
434
+ label="Text to Analyze"
435
+ )
436
+ submit_btn = gr.Button("Analyze Risk", variant="primary")
437
+
438
+ with gr.Column():
439
+ output_json = gr.JSON(label="Detailed Analysis Results")
440
+
441
+ submit_btn.click(fn=predict, inputs=input_text, outputs=output_json)
442
+
443
+ gr.Examples(
444
+ examples=[
445
+ "The moon is made of cheese.",
446
+ "Water boils at 100 degrees Celsius at sea level.",
447
+ "According to a 2031 study from the Institute of Neural Enhancement, humans only use 10% of their brain.",
448
+ "Global temperatures hit record highs in 2024.",
449
+ ],
450
+ inputs=input_text
451
+ )
452
+
453
+ if __name__ == "__main__":
454
+ demo.launch(server_name="0.0.0.0", server_port=7860)