import gradio as gr
import pandas as pd
import warnings
import random
import re
import time
import os
import sys
warnings.filterwarnings('ignore')
# Pre-download NLTK data at startup
import nltk
print("Downloading NLTK data...")
nltk.download('stopwords', quiet=True)
nltk.download('punkt', quiet=True)
print("NLTK data downloaded.")
# Reliable model names and descriptions - replaced KeyBERT with working alternatives
KEYWORD_MODELS = {
'yake_yake': 'YAKE - Yet Another Keyword Extractor (statistical)',
'tfidf_cosine': 'TF-IDF with Cosine Similarity - Document similarity approach',
'rake_nltk': 'RAKE-NLTK - Rapid Automatic Keyword Extraction',
'textrank': 'TextRank - Graph-based ranking algorithm'
}
# Color palette for keywords based on scores
SCORE_COLORS = {
'high': '#00B894', # Green - High relevance
'medium': '#F9CA24', # Yellow - Medium relevance
'low': '#FF6B6B' # Red - Low relevance
}
# Additional colors for variety
KEYWORD_COLORS = [
'#4ECDC4', '#45B7D1', '#6C5CE7', '#A0E7E5', '#FD79A8',
'#8E8E93', '#55A3FF', '#E17055', '#DDA0DD', '#FF9F43',
'#10AC84', '#EE5A24', '#0FBC89', '#5F27CD', '#FF3838'
]
class KeywordExtractionManager:
def __init__(self):
self.rake_extractor = None
self.models_initialized = False
self.initialize_models()
def initialize_models(self):
"""Pre-initialize models to check availability"""
print("Initializing models...")
# Test YAKE
try:
import yake
print("✓ YAKE available")
except ImportError as e:
print(f"✗ YAKE not available: {e}")
# Test RAKE
try:
from rake_nltk import Rake
print("✓ RAKE-NLTK available")
except ImportError as e:
print(f"✗ RAKE-NLTK not available: {e}")
# Test sklearn for TF-IDF
try:
from sklearn.feature_extraction.text import TfidfVectorizer
print("✓ Scikit-learn available for TF-IDF")
except ImportError as e:
print(f"✗ Scikit-learn not available: {e}")
# Test networkx for TextRank
try:
import networkx
print("✓ NetworkX available for TextRank")
except ImportError as e:
print(f"✗ NetworkX not available: {e}")
self.models_initialized = True
def load_rake_extractor(self):
"""Load RAKE extractor with better error handling"""
if self.rake_extractor is None:
try:
from rake_nltk import Rake
# Create RAKE instance
self.rake_extractor = Rake()
print("✓ RAKE extractor loaded successfully")
except Exception as e:
print(f"Error loading RAKE extractor: {str(e)}")
print(f"Full error: {type(e).__name__}: {str(e)}")
return None
return self.rake_extractor
def extract_keywords(self, text, model_name, num_keywords=10, ngram_range=(1, 3), progress=None):
"""Extract keywords using the specified model"""
try:
if progress:
progress(0.3, desc="Loading model...")
print(f"Attempting to extract keywords with {model_name}")
# Handle different model types
if model_name.startswith('yake_'):
return self.extract_yake_keywords(text, num_keywords, ngram_range, progress)
elif model_name.startswith('tfidf_'):
return self.extract_tfidf_cosine_keywords(text, num_keywords, ngram_range, progress)
elif model_name.startswith('rake_'):
return self.extract_rake_keywords(text, num_keywords, progress)
elif model_name.startswith('textrank'):
return self.extract_textrank_keywords(text, num_keywords, ngram_range, progress)
else:
raise ValueError(f"Unknown model: {model_name}")
except Exception as e:
print(f"Error with {model_name}: {str(e)}")
print(f"Full error: {type(e).__name__}: {str(e)}")
return self.fallback_keyword_extraction(text, num_keywords)
def extract_yake_keywords(self, text, num_keywords, ngram_range, progress):
"""Extract keywords using YAKE"""
try:
import yake
if progress:
progress(0.5, desc="Processing with YAKE...")
# Configure YAKE
kw_extractor = yake.KeywordExtractor(
lan="en",
n=ngram_range[1],
dedupLim=0.7,
top=num_keywords
)
if progress:
progress(0.7, desc="Extracting keywords...")
keywords = kw_extractor.extract_keywords(text)
# Format results (YAKE returns lower scores for better keywords)
results = []
for keyword, score in keywords:
# Invert score for consistency (higher = better)
inverted_score = 1.0 / (1.0 + score)
results.append({
'keyword': keyword,
'score': inverted_score,
'model': 'YAKE'
})
print(f"YAKE extracted {len(results)} keywords")
return results
except Exception as e:
print(f"YAKE extraction failed: {type(e).__name__}: {str(e)}")
return self.fallback_keyword_extraction(text, num_keywords)
def extract_tfidf_cosine_keywords(self, text, num_keywords, ngram_range, progress):
"""Extract keywords using TF-IDF with cosine similarity"""
try:
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
if progress:
progress(0.5, desc="Processing with TF-IDF...")
# Create TF-IDF vectorizer
vectorizer = TfidfVectorizer(
ngram_range=ngram_range,
stop_words='english',
max_features=5000,
min_df=1,
max_df=0.95
)
# Extract candidate keywords/phrases
words = re.findall(r'\b[a-z]+\b', text.lower())
candidates = []
# Generate n-grams
for n in range(ngram_range[0], ngram_range[1] + 1):
for i in range(len(words) - n + 1):
candidate = ' '.join(words[i:i+n])
if len(candidate) > 2 and candidate not in candidates:
candidates.append(candidate)
if not candidates:
return self.fallback_keyword_extraction(text, num_keywords)
# Limit candidates to prevent memory issues
candidates = candidates[:300]
if progress:
progress(0.7, desc="Computing similarities...")
try:
# Create document embedding
doc_embedding = vectorizer.fit_transform([text])
# Create embeddings for candidates
candidate_embeddings = vectorizer.transform(candidates)
# Calculate similarities
similarities = cosine_similarity(doc_embedding, candidate_embeddings)[0]
# Get top keywords
top_indices = similarities.argsort()[-num_keywords:][::-1]
results = []
for idx in top_indices:
if similarities[idx] > 0:
results.append({
'keyword': candidates[idx],
'score': float(similarities[idx]),
'model': 'TF-IDF-Cosine'
})
if progress:
progress(0.8, desc="Formatting results...")
print(f"TF-IDF extracted {len(results)} keywords")
return results
except Exception as e:
print(f"TF-IDF approach failed: {e}")
# Fall back to simple TF-IDF
return self.simple_tfidf_extraction(text, num_keywords, ngram_range)
except ImportError:
print("scikit-learn not available for TF-IDF")
return self.fallback_keyword_extraction(text, num_keywords)
except Exception as e:
print(f"TF-IDF extraction failed: {e}")
return self.fallback_keyword_extraction(text, num_keywords)
def extract_textrank_keywords(self, text, num_keywords, ngram_range, progress):
"""Extract keywords using TextRank algorithm"""
try:
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import networkx as nx
if progress:
progress(0.5, desc="Processing with TextRank...")
# Split text into sentences
sentences = re.split(r'[.!?]+', text)
sentences = [s.strip() for s in sentences if s.strip()]
if len(sentences) < 2:
# If text is too short, use simple extraction
return self.simple_tfidf_extraction(text, num_keywords, ngram_range)
# Create TF-IDF matrix
vectorizer = TfidfVectorizer(
ngram_range=(1, 1), # Use unigrams for sentence similarity
stop_words='english'
)
tfidf_matrix = vectorizer.fit_transform(sentences)
# Calculate similarity matrix
similarity_matrix = cosine_similarity(tfidf_matrix)
if progress:
progress(0.6, desc="Building graph...")
# Build graph
nx_graph = nx.from_numpy_array(similarity_matrix)
# Calculate PageRank scores
scores = nx.pagerank(nx_graph)
# Extract keywords from top-ranked sentences
top_sentence_indices = sorted(scores.keys(), key=lambda x: scores[x], reverse=True)[:3]
# Extract keywords from top sentences
keyword_vectorizer = TfidfVectorizer(
ngram_range=ngram_range,
stop_words='english',
max_features=num_keywords * 2
)
top_sentences = [sentences[i] for i in top_sentence_indices]
top_text = ' '.join(top_sentences)
if progress:
progress(0.7, desc="Extracting keywords...")
tfidf_matrix = keyword_vectorizer.fit_transform([top_text])
feature_names = keyword_vectorizer.get_feature_names_out()
tfidf_scores = tfidf_matrix.toarray()[0]
# Get top keywords
top_indices = tfidf_scores.argsort()[-num_keywords:][::-1]
results = []
for idx in top_indices:
if tfidf_scores[idx] > 0:
results.append({
'keyword': feature_names[idx],
'score': float(tfidf_scores[idx]),
'model': 'TextRank'
})
print(f"TextRank extracted {len(results)} keywords")
return results
except ImportError as e:
print(f"Required library not available for TextRank: {e}")
return self.fallback_keyword_extraction(text, num_keywords)
except Exception as e:
print(f"TextRank extraction failed: {e}")
return self.fallback_keyword_extraction(text, num_keywords)
def simple_tfidf_extraction(self, text, num_keywords, ngram_range):
"""Simple TF-IDF extraction without cosine similarity"""
try:
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(
ngram_range=ngram_range,
stop_words='english',
max_features=num_keywords * 2
)
# Fit and transform
tfidf_matrix = vectorizer.fit_transform([text])
# Get feature names and scores
feature_names = vectorizer.get_feature_names_out()
scores = tfidf_matrix.toarray()[0]
# Get top keywords
top_indices = scores.argsort()[-num_keywords:][::-1]
results = []
for idx in top_indices:
if scores[idx] > 0:
results.append({
'keyword': feature_names[idx],
'score': float(scores[idx]),
'model': 'TF-IDF-Simple'
})
return results
except Exception as e:
print(f"Simple TF-IDF failed: {e}")
return self.fallback_keyword_extraction(text, num_keywords)
def extract_rake_keywords(self, text, num_keywords, progress):
"""Extract keywords using RAKE"""
try:
if progress:
progress(0.5, desc="Processing with RAKE...")
rake_extractor = self.load_rake_extractor()
if rake_extractor is None:
print("RAKE extractor could not be loaded")
return self.fallback_keyword_extraction(text, num_keywords)
if progress:
progress(0.7, desc="Extracting keywords...")
# Extract keywords
rake_extractor.extract_keywords_from_text(text)
keywords_with_scores = rake_extractor.get_ranked_phrases_with_scores()
# Normalize scores
if keywords_with_scores:
max_score = max(score for score, _ in keywords_with_scores)
# Format results
results = []
for score, keyword in keywords_with_scores[:num_keywords]:
normalized_score = score / max_score if max_score > 0 else 0
results.append({
'keyword': keyword,
'score': normalized_score,
'model': 'RAKE-NLTK'
})
print(f"RAKE extracted {len(results)} keywords")
return results
else:
print("RAKE returned no keywords")
return self.fallback_keyword_extraction(text, num_keywords)
except Exception as e:
print(f"RAKE extraction failed: {type(e).__name__}: {str(e)}")
return self.fallback_keyword_extraction(text, num_keywords)
def fallback_keyword_extraction(self, text, num_keywords=10):
"""Simple fallback keyword extraction using basic statistics"""
print("Using fallback keyword extraction")
import re
from collections import Counter
# Simple tokenization and filtering
words = re.findall(r'\b[a-z]+\b', text.lower())
# Remove common stop words
stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
'could', 'should', 'may', 'might', 'must', 'can', 'this', 'that',
'these', 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they'}
filtered_words = [w for w in words if w not in stop_words and len(w) > 3]
# Count frequencies
word_freq = Counter(filtered_words)
# Get top keywords
results = []
for word, freq in word_freq.most_common(num_keywords):
score = freq / len(filtered_words) # Normalize by total words
results.append({
'keyword': word,
'score': score,
'model': 'Fallback-TFIDF'
})
return results
def get_score_color(score, max_score):
"""Get color based on score relative to max score"""
if max_score == 0:
return SCORE_COLORS['medium']
relative_score = score / max_score
if relative_score >= 0.7:
return SCORE_COLORS['high']
elif relative_score >= 0.4:
return SCORE_COLORS['medium']
else:
return SCORE_COLORS['low']
def get_relevance_level(score, max_score):
"""Get relevance level name based on score"""
if max_score == 0:
return 'medium'
relative_score = score / max_score
if relative_score >= 0.7:
return 'high'
elif relative_score >= 0.4:
return 'medium'
else:
return 'low'
def create_highlighted_html(text, keywords):
"""Create HTML with highlighted keywords in the text"""
if not keywords:
return f""
# Sort keywords by length (longest first) to avoid partial matches
sorted_keywords = sorted(keywords, key=lambda x: len(x['keyword']), reverse=True)
# Get max score for color scaling
max_score = max(k['score'] for k in keywords) if keywords else 1
# Create a modified text with highlights
highlighted_text = text
for i, kw_data in enumerate(sorted_keywords):
keyword = kw_data['keyword']
score = kw_data['score']
color = get_score_color(score, max_score)
# Create regex pattern for whole word matching (case-insensitive)
pattern = r'\b' + re.escape(keyword) + r'\b'
# Replace with highlighted version
replacement = f'{keyword}'
highlighted_text = re.sub(pattern, replacement, highlighted_text, flags=re.IGNORECASE)
return f"""
📝 Text with Highlighted Keywords
{highlighted_text}
"""
def create_keyword_table_html(keywords):
"""Create HTML table for keywords"""
if not keywords:
return "No keywords found.
"
# Sort by score
sorted_keywords = sorted(keywords, key=lambda x: x['score'], reverse=True)
max_score = sorted_keywords[0]['score'] if sorted_keywords else 1
table_html = """
🎯 Extracted Keywords
| Rank |
Keyword |
Score |
Relevance |
Model |
"""
for i, kw_data in enumerate(sorted_keywords):
score = kw_data['score']
color = get_score_color(score, max_score)
# Create relevance bar
bar_width = int((score / max_score) * 100) if max_score > 0 else 0
relevance_bar = f"""
"""
table_html += f"""
| #{i+1} |
{kw_data['keyword']} |
{score:.4f}
|
{relevance_bar} |
{kw_data['model']}
|
"""
table_html += """
"""
return table_html
def create_legend_html():
"""Create a legend showing score colors"""
html = """
🎨 Relevance Score Legend
High Relevance (70%+)
Medium Relevance (40-70%)
Low Relevance (<40%)
"""
return html
# Initialize the keyword extraction manager
print("Initializing keyword extraction manager...")
keyword_manager = KeywordExtractionManager()
def process_text(text, selected_model, num_keywords, ngram_min, ngram_max, progress=gr.Progress()):
"""Main processing function for Gradio interface with progress tracking"""
if not text.strip():
return "❌ Please enter some text to analyse", "", ""
progress(0.1, desc="Initialising...")
# Extract keywords
progress(0.2, desc="Extracting keywords...")
keywords = keyword_manager.extract_keywords(
text,
selected_model,
num_keywords=num_keywords,
ngram_range=(ngram_min, ngram_max),
progress=progress
)
if not keywords:
return "❌ No keywords found. Try adjusting the parameters.", "", ""
progress(0.8, desc="Processing results...")
# Create outputs
legend_html = create_legend_html()
highlighted_html = create_highlighted_html(text, keywords)
results_html = create_keyword_table_html(keywords)
progress(0.9, desc="Creating summary...")
# Create summary
avg_score = sum(k['score'] for k in keywords) / len(keywords)
model_display = selected_model.replace('yake_', '').replace('tfidf_', 'TF-IDF ').replace('rake_', 'RAKE-').replace('textrank', 'TextRank').title()
summary = f"""
## 📊 Analysis Summary
- **Keywords extracted:** {len(keywords)}
- **Model used:** {model_display}
- **Average relevance score:** {avg_score:.4f}
- **N-gram range:** {ngram_min}-{ngram_max} words
"""
progress(1.0, desc="Complete!")
return summary, legend_html + highlighted_html, results_html
# Create Gradio interface
def create_interface():
# Note: theme moved to launch() for Gradio 6.0 compatibility
with gr.Blocks(title="Keyword Extraction Tool") as demo:
gr.Markdown("""
# Keyword Extraction Explorer Tool
Extract the most important keywords and phrases from your text using various algorithms! This tool uses modern keyword extraction methods including YAKE, TF-IDF, RAKE, and TextRank for comprehensive analysis.
### How to use:
1. **📝 Enter your text** in the text area below
2. **🎯 Select a model** from the dropdown for keyword extraction
3. **⚙️ Adjust parameters** (number of keywords, n-gram range)
4. **🔍 Click "Extract Keywords"** to see results with organized output
""")
# Add tip box
gr.HTML("""
💡 Top tip: Different models excel at different types of texts - experiment to find the best one for your content!
""")
with gr.Row():
with gr.Column(scale=2):
text_input = gr.Textbox(
label="📝 Text to Analyse",
placeholder="Enter your text here...",
lines=18,
max_lines=20
)
with gr.Column(scale=1):
# Model selector
model_dropdown = gr.Dropdown(
choices=list(KEYWORD_MODELS.keys()),
value='yake_yake',
label="🎯 Select Keyword Extraction Model"
)
# Parameters
num_keywords = gr.Slider(
minimum=5,
maximum=30,
value=10,
step=1,
label="📊 Number of Keywords"
)
with gr.Row():
ngram_min = gr.Slider(
minimum=1,
maximum=3,
value=1,
step=1,
label="Min N-gram"
)
ngram_max = gr.Slider(
minimum=1,
maximum=4,
value=3,
step=1,
label="Max N-gram"
)
# Add N-gram tip box
gr.HTML("""
💡 Top tip: N-grams are sequences of words. Set Min=1, Max=3 to extract single words, phrases of 2 words, and phrases of 3 words. Higher values capture longer phrases but may reduce precision.
""")
# Add model descriptions
gr.HTML("""
ℹ️ Model Descriptions
- YAKE:
- Statistical approach requiring no training - works well on short texts and multilingual content
- TF-IDF with Cosine Similarity:
- Document similarity approach - extracts keywords most similar to the overall document
- RAKE-NLTK:
- Classic keyword extraction algorithm - fast and reliable for phrase extraction
- TextRank:
- Graph-based ranking algorithm inspired by PageRank - good for extracting key concepts
""")
extract_btn = gr.Button("🔍 Extract Keywords", variant="primary", size="lg")
# Output sections
with gr.Row():
summary_output = gr.Markdown(label="Summary")
with gr.Row():
highlighted_output = gr.HTML(label="Highlighted Text")
# Results section
with gr.Row():
with gr.Column():
gr.Markdown("### 📋 Detailed Results")
results_output = gr.HTML(label="Keyword Results")
# Connect the button to the processing function
extract_btn.click(
fn=process_text,
inputs=[
text_input,
model_dropdown,
num_keywords,
ngram_min,
ngram_max
],
outputs=[summary_output, highlighted_output, results_output]
)
gr.Examples(
examples=[
[
"On June 6, 1944, Allied forces launched Operation Overlord, the invasion of Normandy. General Dwight D. Eisenhower commanded the operation, while Field Marshal Bernard Montgomery led ground forces. The BBC broadcast coded messages to the French Resistance, including the famous line 'The long sobs of autumn violins.'",
"yake_yake",
10,
1,
3
],
[
"In Jane Austen's 'Pride and Prejudice', Elizabeth Bennet first meets Mr. Darcy at the Meryton assembly. The novel, published in 1813, explores themes of marriage and social class in Regency England. Austen wrote to her sister Cassandra about the manuscript while staying at Chawton Cottage.",
"tfidf_cosine",
10,
1,
3
],
[
"Charles Darwin arrived at the Galápagos Islands aboard HMS Beagle in September 1835. During his five-week visit, Darwin collected specimens of finches, tortoises, and mockingbirds. His observations of these species' variations across different islands later contributed to his theory of evolution by natural selection, published in 'On the Origin of Species' in 1859.",
"rake_nltk",
10,
1,
3
]
],
inputs=[
text_input,
model_dropdown,
num_keywords,
ngram_min,
ngram_max
]
)
# Add model information links
gr.HTML("""
📚 Model Information & Documentation
Learn more about the algorithms used in this tool:
This Keyword Extraction Explorer Tool was created as part of the
Digital Scholarship at Oxford (DiSc)
funded research project:
Extracting Keywords from Crowdsourced Collections.
The code for this tool was built with the aid of Claude Opus 4.
""")
return demo
if __name__ == "__main__":
demo = create_interface()
# Gradio 6.0 compatibility: theme and ssr_mode moved to launch()
demo.launch(ssr_mode=False, theme=gr.themes.Soft())