Update app.py
Browse files
app.py
CHANGED
|
@@ -1,1113 +1,251 @@
|
|
| 1 |
import gradio as gr
|
| 2 |
-
import torch
|
| 3 |
-
from gliner import GLiNER
|
| 4 |
-
import pandas as pd
|
| 5 |
-
import warnings
|
| 6 |
-
import random
|
| 7 |
import re
|
| 8 |
-
import
|
| 9 |
-
warnings.filterwarnings('ignore')
|
| 10 |
-
|
| 11 |
-
# Common NER entity types (using full names)
|
| 12 |
-
STANDARD_ENTITIES = [
|
| 13 |
-
'DATE', 'EVENT', 'FACILITY', 'GEOPOLITICAL ENTITY', 'LANGUAGE', 'LOCATION',
|
| 14 |
-
'MISCELLANEOUS', 'NATIONALITIES/GROUPS', 'ORGANIZATION', 'PERSON', 'PRODUCT', 'WORK OF ART'
|
| 15 |
-
]
|
| 16 |
-
|
| 17 |
-
# Colour schemes (updated to match full names)
|
| 18 |
-
STANDARD_COLORS = {
|
| 19 |
-
'DATE': '#FF6B6B', # Red
|
| 20 |
-
'EVENT': '#4ECDC4', # Teal
|
| 21 |
-
'FACILITY': '#45B7D1', # Blue
|
| 22 |
-
'GEOPOLITICAL ENTITY': '#F9CA24', # Yellow
|
| 23 |
-
'LANGUAGE': '#6C5CE7', # Purple
|
| 24 |
-
'LOCATION': '#A0E7E5', # Light Cyan
|
| 25 |
-
'MISCELLANEOUS': '#FD79A8', # Pink
|
| 26 |
-
'NATIONALITIES/GROUPS': '#8E8E93', # Grey
|
| 27 |
-
'ORGANIZATION': '#55A3FF', # Light Blue
|
| 28 |
-
'PERSON': '#00B894', # Green
|
| 29 |
-
'PRODUCT': '#E17055', # Orange-Red
|
| 30 |
-
'WORK OF ART': '#DDA0DD' # Plum
|
| 31 |
-
}
|
| 32 |
-
|
| 33 |
-
# Entity definitions for glossary (alphabetically ordered with full name (abbreviation) format)
|
| 34 |
-
ENTITY_DEFINITIONS = {
|
| 35 |
-
'DATE': 'Date (DATE): Absolute or relative dates or periods',
|
| 36 |
-
'EVENT': 'Event (EVENT): Named hurricanes, battles, wars, sports events, etc.',
|
| 37 |
-
'FACILITY': 'Facility (FAC): Buildings, airports, highways, bridges, etc.',
|
| 38 |
-
'GEOPOLITICAL ENTITY': 'Geopolitical Entity (GPE): Countries, cities, states',
|
| 39 |
-
'LANGUAGE': 'Language (LANG): Any named language',
|
| 40 |
-
'LOCATION': 'Location (LOC): Non-GPE locations - Mountain ranges, bodies of water',
|
| 41 |
-
'MISCELLANEOUS': 'Miscellaneous (MISC): Entities that don\'t fit elsewhere',
|
| 42 |
-
'NATIONALITIES/GROUPS': 'Nationalities/Groups (NORP): Nationalities or religious or political groups',
|
| 43 |
-
'ORGANIZATION': 'Organization (ORG): Companies, agencies, institutions, etc.',
|
| 44 |
-
'PERSON': 'Person (PER): People, including fictional characters',
|
| 45 |
-
'PRODUCT': 'Product (PRODUCT): Objects, vehicles, foods, etc. (Not services)',
|
| 46 |
-
'WORK OF ART': 'Work of Art (Work of Art): Titles of books, songs, movies, paintings, etc.'
|
| 47 |
-
}
|
| 48 |
-
|
| 49 |
-
# Additional colours for custom entities
|
| 50 |
-
CUSTOM_COLOR_PALETTE = [
|
| 51 |
-
'#FF9F43', '#10AC84', '#EE5A24', '#0FBC89', '#5F27CD',
|
| 52 |
-
'#FF3838', '#2F3640', '#3742FA', '#2ED573', '#FFA502',
|
| 53 |
-
'#FF6348', '#1E90FF', '#FF1493', '#32CD32', '#FFD700',
|
| 54 |
-
'#FF4500', '#DA70D6', '#00CED1', '#FF69B4', '#7B68EE'
|
| 55 |
-
]
|
| 56 |
-
|
| 57 |
-
class HybridNERManager:
|
| 58 |
-
def __init__(self):
|
| 59 |
-
self.gliner_model = None
|
| 60 |
-
self.spacy_model = None
|
| 61 |
-
self.flair_models = {}
|
| 62 |
-
self.all_entity_colors = {}
|
| 63 |
-
self.model_names = [
|
| 64 |
-
'flair_ner-large',
|
| 65 |
-
'spacy_en_core_web_trf',
|
| 66 |
-
'flair_ner-ontonotes-large',
|
| 67 |
-
'gliner_knowledgator/modern-gliner-bi-large-v1.0'
|
| 68 |
-
]
|
| 69 |
-
# Mapping from full names to abbreviations for model compatibility
|
| 70 |
-
self.entity_mapping = {
|
| 71 |
-
'DATE': 'DATE',
|
| 72 |
-
'EVENT': 'EVENT',
|
| 73 |
-
'FACILITY': 'FAC',
|
| 74 |
-
'GEOPOLITICAL ENTITY': 'GPE',
|
| 75 |
-
'LANGUAGE': 'LANG',
|
| 76 |
-
'LOCATION': 'LOC',
|
| 77 |
-
'MISCELLANEOUS': 'MISC',
|
| 78 |
-
'NATIONALITIES/GROUPS': 'NORP',
|
| 79 |
-
'ORGANIZATION': 'ORG',
|
| 80 |
-
'PERSON': 'PER',
|
| 81 |
-
'PRODUCT': 'PRODUCT',
|
| 82 |
-
'WORK OF ART': 'Work of Art'
|
| 83 |
-
}
|
| 84 |
-
# Reverse mapping for display
|
| 85 |
-
self.abbrev_to_full = {v: k for k, v in self.entity_mapping.items()}
|
| 86 |
-
|
| 87 |
-
def load_model(self, model_name):
|
| 88 |
-
"""Load the specified model"""
|
| 89 |
-
try:
|
| 90 |
-
if 'spacy' in model_name:
|
| 91 |
-
return self.load_spacy_model()
|
| 92 |
-
elif 'flair' in model_name:
|
| 93 |
-
return self.load_flair_model(model_name)
|
| 94 |
-
elif 'gliner' in model_name:
|
| 95 |
-
return self.load_gliner_model()
|
| 96 |
-
except Exception as e:
|
| 97 |
-
print(f"Error loading {model_name}: {str(e)}")
|
| 98 |
-
return None
|
| 99 |
-
|
| 100 |
-
def load_spacy_model(self):
|
| 101 |
-
"""Load spaCy model for common NER"""
|
| 102 |
-
if self.spacy_model is None:
|
| 103 |
-
try:
|
| 104 |
-
import spacy
|
| 105 |
-
try:
|
| 106 |
-
# Try transformer model first, fallback to small model
|
| 107 |
-
self.spacy_model = spacy.load("en_core_web_trf")
|
| 108 |
-
print("✓ spaCy transformer model loaded successfully")
|
| 109 |
-
except OSError:
|
| 110 |
-
try:
|
| 111 |
-
self.spacy_model = spacy.load("en_core_web_sm")
|
| 112 |
-
print("✓ spaCy common model loaded successfully")
|
| 113 |
-
except OSError:
|
| 114 |
-
print("spaCy model not found. Using GLiNER for all entity types.")
|
| 115 |
-
return None
|
| 116 |
-
except Exception as e:
|
| 117 |
-
print(f"Error loading spaCy model: {str(e)}")
|
| 118 |
-
return None
|
| 119 |
-
return self.spacy_model
|
| 120 |
-
|
| 121 |
-
def load_flair_model(self, model_name):
|
| 122 |
-
"""Load Flair models"""
|
| 123 |
-
if model_name not in self.flair_models:
|
| 124 |
-
try:
|
| 125 |
-
from flair.models import SequenceTagger
|
| 126 |
-
if 'ontonotes' in model_name:
|
| 127 |
-
model = SequenceTagger.load("flair/ner-english-ontonotes-large")
|
| 128 |
-
print("✓ Flair OntoNotes model loaded successfully")
|
| 129 |
-
else:
|
| 130 |
-
model = SequenceTagger.load("flair/ner-english-large")
|
| 131 |
-
print("✓ Flair large model loaded successfully")
|
| 132 |
-
self.flair_models[model_name] = model
|
| 133 |
-
except Exception as e:
|
| 134 |
-
print(f"Error loading {model_name}: {str(e)}")
|
| 135 |
-
# Fallback to GLiNER
|
| 136 |
-
return self.load_gliner_model()
|
| 137 |
-
return self.flair_models[model_name]
|
| 138 |
-
|
| 139 |
-
def load_gliner_model(self):
|
| 140 |
-
"""Load GLiNER model for custom entities"""
|
| 141 |
-
if self.gliner_model is None:
|
| 142 |
-
try:
|
| 143 |
-
# Try the modern GLiNER model first, fallback to stable model
|
| 144 |
-
self.gliner_model = GLiNER.from_pretrained("knowledgator/gliner-bi-large-v1.0")
|
| 145 |
-
print("✓ GLiNER knowledgator model loaded successfully")
|
| 146 |
-
except Exception as e:
|
| 147 |
-
print(f"Primary GLiNER model failed: {str(e)}")
|
| 148 |
-
try:
|
| 149 |
-
# Fallback to stable model
|
| 150 |
-
self.gliner_model = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
|
| 151 |
-
print("✓ GLiNER fallback model loaded successfully")
|
| 152 |
-
except Exception as e2:
|
| 153 |
-
print(f"Error loading GLiNER model: {str(e2)}")
|
| 154 |
-
return None
|
| 155 |
-
return self.gliner_model
|
| 156 |
-
|
| 157 |
-
def assign_colours(self, standard_entities, custom_entities):
|
| 158 |
-
"""Assign colours to all entity types"""
|
| 159 |
-
self.all_entity_colors = {}
|
| 160 |
-
|
| 161 |
-
# Assign common colours
|
| 162 |
-
for entity in standard_entities:
|
| 163 |
-
self.all_entity_colors[entity.upper()] = STANDARD_COLORS.get(entity.upper(), '#CCCCCC')
|
| 164 |
-
|
| 165 |
-
# Assign custom colours
|
| 166 |
-
for i, entity in enumerate(custom_entities):
|
| 167 |
-
if i < len(CUSTOM_COLOR_PALETTE):
|
| 168 |
-
self.all_entity_colors[entity.upper()] = CUSTOM_COLOR_PALETTE[i]
|
| 169 |
-
else:
|
| 170 |
-
# Generate random colour if we run out
|
| 171 |
-
self.all_entity_colors[entity.upper()] = f"#{random.randint(0, 0xFFFFFF):06x}"
|
| 172 |
-
|
| 173 |
-
return self.all_entity_colors
|
| 174 |
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
|
|
|
|
|
|
|
|
|
| 184 |
|
| 185 |
-
if 'spacy' in model_name:
|
| 186 |
-
return self.extract_spacy_entities(text, abbrev_types)
|
| 187 |
-
elif 'flair' in model_name:
|
| 188 |
-
return self.extract_flair_entities(text, abbrev_types, model_name)
|
| 189 |
-
elif 'gliner' in model_name:
|
| 190 |
-
return self.extract_gliner_entities(text, abbrev_types, threshold, is_custom=False)
|
| 191 |
-
else:
|
| 192 |
-
return []
|
| 193 |
-
|
| 194 |
-
def extract_spacy_entities(self, text, entity_types):
|
| 195 |
-
"""Extract entities using spaCy"""
|
| 196 |
-
model = self.load_spacy_model()
|
| 197 |
-
if model is None:
|
| 198 |
-
return []
|
| 199 |
-
|
| 200 |
-
try:
|
| 201 |
-
doc = model(text)
|
| 202 |
-
entities = []
|
| 203 |
-
for ent in doc.ents:
|
| 204 |
-
if ent.label_ in entity_types:
|
| 205 |
-
# Convert abbreviation back to full name for display
|
| 206 |
-
display_label = self.abbrev_to_full.get(ent.label_, ent.label_)
|
| 207 |
-
entities.append({
|
| 208 |
-
'text': ent.text,
|
| 209 |
-
'label': display_label,
|
| 210 |
-
'start': ent.start_char,
|
| 211 |
-
'end': ent.end_char,
|
| 212 |
-
'confidence': 1.0, # spaCy doesn't provide confidence scores
|
| 213 |
-
'source': 'spaCy'
|
| 214 |
-
})
|
| 215 |
-
return entities
|
| 216 |
-
except Exception as e:
|
| 217 |
-
print(f"Error with spaCy extraction: {str(e)}")
|
| 218 |
-
return []
|
| 219 |
-
|
| 220 |
-
def extract_flair_entities(self, text, entity_types, model_name):
|
| 221 |
-
"""Extract entities using Flair"""
|
| 222 |
-
model = self.load_flair_model(model_name)
|
| 223 |
-
if model is None:
|
| 224 |
-
return []
|
| 225 |
-
|
| 226 |
-
try:
|
| 227 |
-
from flair.data import Sentence
|
| 228 |
-
sentence = Sentence(text)
|
| 229 |
-
model.predict(sentence)
|
| 230 |
-
entities = []
|
| 231 |
-
for entity in sentence.get_spans('ner'):
|
| 232 |
-
# Map Flair labels to our common set
|
| 233 |
-
label = entity.tag
|
| 234 |
-
if label == 'PERSON':
|
| 235 |
-
label = 'PER'
|
| 236 |
-
elif label == 'ORGANIZATION':
|
| 237 |
-
label = 'ORG'
|
| 238 |
-
elif label == 'LOCATION':
|
| 239 |
-
label = 'LOC'
|
| 240 |
-
elif label == 'MISCELLANEOUS':
|
| 241 |
-
label = 'MISC'
|
| 242 |
-
|
| 243 |
-
if label in entity_types:
|
| 244 |
-
# Convert abbreviation back to full name for display
|
| 245 |
-
display_label = self.abbrev_to_full.get(label, label)
|
| 246 |
-
entities.append({
|
| 247 |
-
'text': entity.text,
|
| 248 |
-
'label': display_label,
|
| 249 |
-
'start': entity.start_position,
|
| 250 |
-
'end': entity.end_position,
|
| 251 |
-
'confidence': entity.score,
|
| 252 |
-
'source': f'Flair-{model_name.split("-")[-1]}'
|
| 253 |
-
})
|
| 254 |
-
return entities
|
| 255 |
-
except Exception as e:
|
| 256 |
-
print(f"Error with Flair extraction: {str(e)}")
|
| 257 |
-
return []
|
| 258 |
-
|
| 259 |
-
def extract_gliner_entities(self, text, entity_types, threshold=0.3, is_custom=True):
|
| 260 |
-
"""Extract entities using GLiNER"""
|
| 261 |
-
model = self.load_gliner_model()
|
| 262 |
-
if model is None:
|
| 263 |
-
return []
|
| 264 |
-
|
| 265 |
try:
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
for
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
result.append({
|
| 276 |
-
'text': entity['text'],
|
| 277 |
-
'label': display_label,
|
| 278 |
-
'start': entity['start'],
|
| 279 |
-
'end': entity['end'],
|
| 280 |
-
'confidence': entity.get('score', 0.0),
|
| 281 |
-
'source': 'GLiNER-Custom' if is_custom else 'GLiNER-Common'
|
| 282 |
-
})
|
| 283 |
-
return result
|
| 284 |
-
except Exception as e:
|
| 285 |
-
print(f"Error with GLiNER extraction: {str(e)}")
|
| 286 |
-
return []
|
| 287 |
-
|
| 288 |
-
def find_overlapping_entities(entities):
|
| 289 |
-
"""Find and share overlapping entities - specifically entities found by BOTH common NER models AND custom entities"""
|
| 290 |
-
if not entities:
|
| 291 |
-
return []
|
| 292 |
-
|
| 293 |
-
# Sort entities by start position
|
| 294 |
-
sorted_entities = sorted(entities, key=lambda x: x['start'])
|
| 295 |
-
shared_entities = []
|
| 296 |
-
|
| 297 |
-
i = 0
|
| 298 |
-
while i < len(sorted_entities):
|
| 299 |
-
current_entity = sorted_entities[i]
|
| 300 |
-
overlapping_entities = [current_entity]
|
| 301 |
-
|
| 302 |
-
# Find all entities that overlap with current entity
|
| 303 |
-
j = i + 1
|
| 304 |
-
while j < len(sorted_entities):
|
| 305 |
-
next_entity = sorted_entities[j]
|
| 306 |
-
|
| 307 |
-
# Check if entities overlap (same text span or overlapping positions)
|
| 308 |
-
if (current_entity['start'] <= next_entity['start'] < current_entity['end'] or
|
| 309 |
-
next_entity['start'] <= current_entity['start'] < current_entity['end'] or
|
| 310 |
-
current_entity['text'].lower() == next_entity['text'].lower()):
|
| 311 |
-
overlapping_entities.append(next_entity)
|
| 312 |
-
sorted_entities.pop(j)
|
| 313 |
-
else:
|
| 314 |
-
j += 1
|
| 315 |
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
else:
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
has_custom = True
|
| 330 |
-
|
| 331 |
-
if has_common and has_custom:
|
| 332 |
-
# This is a true shared entity (common + custom)
|
| 333 |
-
shared_entity = share_entities(overlapping_entities)
|
| 334 |
-
shared_entities.append(shared_entity)
|
| 335 |
else:
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
i += 1
|
| 340 |
-
|
| 341 |
-
return shared_entities
|
| 342 |
-
|
| 343 |
-
def share_entities(entity_list):
|
| 344 |
-
"""Share multiple overlapping entities into one"""
|
| 345 |
-
if len(entity_list) == 1:
|
| 346 |
-
return entity_list[0]
|
| 347 |
-
|
| 348 |
-
# Use the entity with the longest text span as the base
|
| 349 |
-
base_entity = max(entity_list, key=lambda x: len(x['text']))
|
| 350 |
-
|
| 351 |
-
# Collect all labels and sources
|
| 352 |
-
labels = [entity['label'] for entity in entity_list]
|
| 353 |
-
sources = [entity['source'] for entity in entity_list]
|
| 354 |
-
confidences = [entity['confidence'] for entity in entity_list]
|
| 355 |
-
|
| 356 |
-
return {
|
| 357 |
-
'text': base_entity['text'],
|
| 358 |
-
'start': base_entity['start'],
|
| 359 |
-
'end': base_entity['end'],
|
| 360 |
-
'labels': labels,
|
| 361 |
-
'sources': sources,
|
| 362 |
-
'confidences': confidences,
|
| 363 |
-
'is_shared': True,
|
| 364 |
-
'entity_count': len(entity_list)
|
| 365 |
-
}
|
| 366 |
-
|
| 367 |
-
def create_highlighted_html(text, entities, entity_colors):
|
| 368 |
-
"""Create HTML with highlighted entities"""
|
| 369 |
-
if not entities:
|
| 370 |
-
return f"<div style='padding: 15px; border: 1px solid #ddd; border-radius: 5px; background-color: #fafafa;'><p>{text}</p></div>"
|
| 371 |
-
|
| 372 |
-
# Find and share overlapping entities
|
| 373 |
-
shared_entities = find_overlapping_entities(entities)
|
| 374 |
-
|
| 375 |
-
# Sort by start position
|
| 376 |
-
sorted_entities = sorted(shared_entities, key=lambda x: x['start'])
|
| 377 |
-
|
| 378 |
-
# Create HTML with highlighting
|
| 379 |
-
html_parts = []
|
| 380 |
-
last_end = 0
|
| 381 |
-
|
| 382 |
-
for entity in sorted_entities:
|
| 383 |
-
# Add text before entity
|
| 384 |
-
html_parts.append(text[last_end:entity['start']])
|
| 385 |
-
|
| 386 |
-
if entity.get('is_shared', False):
|
| 387 |
-
# Handle shared entity with multiple colours
|
| 388 |
-
html_parts.append(create_shared_entity_html(entity, entity_colors))
|
| 389 |
-
else:
|
| 390 |
-
# Handle single entity
|
| 391 |
-
html_parts.append(create_single_entity_html(entity, entity_colors))
|
| 392 |
-
|
| 393 |
-
last_end = entity['end']
|
| 394 |
-
|
| 395 |
-
# Add remaining text
|
| 396 |
-
html_parts.append(text[last_end:])
|
| 397 |
-
|
| 398 |
-
highlighted_text = ''.join(html_parts)
|
| 399 |
-
|
| 400 |
-
return f"""
|
| 401 |
-
<div style='padding: 15px; border: 2px solid #ddd; border-radius: 8px; background-color: #fafafa; margin: 10px 0;'>
|
| 402 |
-
<h4 style='margin: 0 0 15px 0; color: #333;'>📝 Text with Highlighted Entities</h4>
|
| 403 |
-
<div style='line-height: 1.8; font-size: 16px; background-color: white; padding: 15px; border-radius: 5px;'>{highlighted_text}</div>
|
| 404 |
-
</div>
|
| 405 |
-
"""
|
| 406 |
-
|
| 407 |
-
def create_single_entity_html(entity, entity_colors):
|
| 408 |
-
"""Create HTML for a single entity"""
|
| 409 |
-
label = entity['label']
|
| 410 |
-
colour = entity_colors.get(label.upper(), '#CCCCCC')
|
| 411 |
-
confidence = entity.get('confidence', 0.0)
|
| 412 |
-
source = entity.get('source', 'Unknown')
|
| 413 |
-
|
| 414 |
-
return (f'<span style="background-color: {colour}; padding: 2px 4px; '
|
| 415 |
-
f'border-radius: 3px; margin: 0 1px; '
|
| 416 |
-
f'border: 1px solid {colour}; color: white; font-weight: bold;" '
|
| 417 |
-
f'title="{label} ({source}) - confidence: {confidence:.2f}">'
|
| 418 |
-
f'{entity["text"]}</span>')
|
| 419 |
-
|
| 420 |
-
def create_shared_entity_html(entity, entity_colors):
|
| 421 |
-
"""Create HTML for a shared entity with multiple colours"""
|
| 422 |
-
labels = entity['labels']
|
| 423 |
-
sources = entity['sources']
|
| 424 |
-
confidences = entity['confidences']
|
| 425 |
-
|
| 426 |
-
# Get colours for each label
|
| 427 |
-
colours = []
|
| 428 |
-
for label in labels:
|
| 429 |
-
colour = entity_colors.get(label.upper(), '#CCCCCC')
|
| 430 |
-
colours.append(colour)
|
| 431 |
-
|
| 432 |
-
# Create gradient background
|
| 433 |
-
if len(colours) == 2:
|
| 434 |
-
gradient = f"linear-gradient(to right, {colours[0]} 50%, {colours[1]} 50%)"
|
| 435 |
-
else:
|
| 436 |
-
# For more colours, create equal segments
|
| 437 |
-
segment_size = 100 / len(colours)
|
| 438 |
-
gradient_parts = []
|
| 439 |
-
for i, colour in enumerate(colours):
|
| 440 |
-
start = i * segment_size
|
| 441 |
-
end = (i + 1) * segment_size
|
| 442 |
-
gradient_parts.append(f"{colour} {start}%, {colour} {end}%")
|
| 443 |
-
gradient = f"linear-gradient(to right, {', '.join(gradient_parts)})"
|
| 444 |
-
|
| 445 |
-
# Create tooltip
|
| 446 |
-
tooltip_parts = []
|
| 447 |
-
for i, label in enumerate(labels):
|
| 448 |
-
tooltip_parts.append(f"{label} ({sources[i]}) - {confidences[i]:.2f}")
|
| 449 |
-
tooltip = " | ".join(tooltip_parts)
|
| 450 |
-
|
| 451 |
-
return (f'<span style="background: {gradient}; padding: 2px 4px; '
|
| 452 |
-
f'border-radius: 3px; margin: 0 1px; '
|
| 453 |
-
f'border: 2px solid #333; color: white; font-weight: bold;" '
|
| 454 |
-
f'title="SHARED: {tooltip}">'
|
| 455 |
-
f'{entity["text"]} 🤝</span>')
|
| 456 |
-
|
| 457 |
-
def create_entity_table_html(entities_of_type, entity_type, colour, is_shared=False):
|
| 458 |
-
"""Create HTML table for a specific entity type"""
|
| 459 |
-
if is_shared:
|
| 460 |
-
table_html = f"""
|
| 461 |
-
<table style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
|
| 462 |
-
<thead>
|
| 463 |
-
<tr style="background-color: {colour}; color: white;">
|
| 464 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Entity Text</th>
|
| 465 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">All Labels</th>
|
| 466 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Sources</th>
|
| 467 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Count</th>
|
| 468 |
-
</tr>
|
| 469 |
-
</thead>
|
| 470 |
-
<tbody>
|
| 471 |
-
"""
|
| 472 |
-
|
| 473 |
-
for entity in entities_of_type:
|
| 474 |
-
labels_text = " | ".join(entity['labels'])
|
| 475 |
-
sources_text = " | ".join(entity['sources'])
|
| 476 |
-
|
| 477 |
-
table_html += f"""
|
| 478 |
-
<tr style="background-color: #fff;">
|
| 479 |
-
<td style="padding: 10px; border: 1px solid #ddd; font-weight: bold;">{entity['text']}</td>
|
| 480 |
-
<td style="padding: 10px; border: 1px solid #ddd;">{labels_text}</td>
|
| 481 |
-
<td style="padding: 10px; border: 1px solid #ddd;">{sources_text}</td>
|
| 482 |
-
<td style="padding: 10px; border: 1px solid #ddd; text-align: center;">
|
| 483 |
-
<span style='background-color: #28a745; color: white; padding: 2px 6px; border-radius: 10px; font-size: 11px;'>
|
| 484 |
-
{entity['entity_count']}
|
| 485 |
-
</span>
|
| 486 |
-
</td>
|
| 487 |
-
</tr>
|
| 488 |
-
"""
|
| 489 |
-
else:
|
| 490 |
-
table_html = f"""
|
| 491 |
-
<table style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
|
| 492 |
-
<thead>
|
| 493 |
-
<tr style="background-color: {colour}; color: white;">
|
| 494 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Entity Text</th>
|
| 495 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Confidence</th>
|
| 496 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Type</th>
|
| 497 |
-
<th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Source</th>
|
| 498 |
-
</tr>
|
| 499 |
-
</thead>
|
| 500 |
-
<tbody>
|
| 501 |
-
"""
|
| 502 |
-
|
| 503 |
-
# Sort by confidence score
|
| 504 |
-
entities_of_type.sort(key=lambda x: x.get('confidence', 0), reverse=True)
|
| 505 |
-
|
| 506 |
-
for entity in entities_of_type:
|
| 507 |
-
confidence = entity.get('confidence', 0.0)
|
| 508 |
-
confidence_colour = "#28a745" if confidence > 0.7 else "#ffc107" if confidence > 0.4 else "#dc3545"
|
| 509 |
-
source = entity.get('source', 'Unknown')
|
| 510 |
-
source_badge = f"<span style='background-color: #007bff; color: white; padding: 2px 6px; border-radius: 10px; font-size: 11px;'>{source}</span>"
|
| 511 |
-
|
| 512 |
-
table_html += f"""
|
| 513 |
-
<tr style="background-color: #fff;">
|
| 514 |
-
<td style="padding: 10px; border: 1px solid #ddd; font-weight: bold;">{entity['text']}</td>
|
| 515 |
-
<td style="padding: 10px; border: 1px solid #ddd;">
|
| 516 |
-
<span style="color: {confidence_colour}; font-weight: bold;">
|
| 517 |
-
{confidence:.3f}
|
| 518 |
-
</span>
|
| 519 |
-
</td>
|
| 520 |
-
<td style="padding: 10px; border: 1px solid #ddd;">{entity['label']}</td>
|
| 521 |
-
<td style="padding: 10px; border: 1px solid #ddd;">{source_badge}</td>
|
| 522 |
-
</tr>
|
| 523 |
-
"""
|
| 524 |
|
| 525 |
-
|
| 526 |
-
return table_html
|
| 527 |
-
|
| 528 |
-
def create_all_entity_tables(entities, entity_colors):
|
| 529 |
-
"""Create all entity tables in a single container"""
|
| 530 |
-
if not entities:
|
| 531 |
-
return "<p style='text-align: center; padding: 20px;'>No entities found.</p>"
|
| 532 |
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
entity_groups = {}
|
| 538 |
-
for entity in shared_entities:
|
| 539 |
-
if entity.get('is_shared', False):
|
| 540 |
-
key = 'SHARED_ENTITIES'
|
| 541 |
-
else:
|
| 542 |
-
key = entity['label']
|
| 543 |
-
|
| 544 |
-
if key not in entity_groups:
|
| 545 |
-
entity_groups[key] = []
|
| 546 |
-
entity_groups[key].append(entity)
|
| 547 |
-
|
| 548 |
-
if not entity_groups:
|
| 549 |
-
return "<p style='text-align: center; padding: 20px;'>No entities found.</p>"
|
| 550 |
-
|
| 551 |
-
# Create container with all tables
|
| 552 |
-
all_tables_html = """
|
| 553 |
-
<div style='max-height: 600px; overflow-y: auto; border: 2px solid #ddd; border-radius: 8px; padding: 20px; background-color: #fafafa;'>
|
| 554 |
-
<style>
|
| 555 |
-
.entity-section {
|
| 556 |
-
margin-bottom: 30px;
|
| 557 |
-
background-color: white;
|
| 558 |
-
border-radius: 8px;
|
| 559 |
-
padding: 15px;
|
| 560 |
-
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
| 561 |
-
}
|
| 562 |
-
.entity-section:last-child {
|
| 563 |
-
margin-bottom: 0;
|
| 564 |
-
}
|
| 565 |
-
.section-header {
|
| 566 |
-
display: flex;
|
| 567 |
-
align-items: center;
|
| 568 |
-
margin-bottom: 15px;
|
| 569 |
-
padding-bottom: 10px;
|
| 570 |
-
border-bottom: 2px solid #eee;
|
| 571 |
-
}
|
| 572 |
-
.entity-count {
|
| 573 |
-
background-color: #007bff;
|
| 574 |
-
color: white;
|
| 575 |
-
padding: 4px 12px;
|
| 576 |
-
border-radius: 15px;
|
| 577 |
-
font-size: 14px;
|
| 578 |
-
font-weight: bold;
|
| 579 |
-
margin-left: 10px;
|
| 580 |
-
}
|
| 581 |
-
.quick-nav {
|
| 582 |
-
position: sticky;
|
| 583 |
-
top: 0;
|
| 584 |
-
background-color: #f8f9fa;
|
| 585 |
-
padding: 10px;
|
| 586 |
-
margin-bottom: 20px;
|
| 587 |
-
border-radius: 8px;
|
| 588 |
-
display: flex;
|
| 589 |
-
flex-wrap: wrap;
|
| 590 |
-
gap: 8px;
|
| 591 |
-
z-index: 10;
|
| 592 |
-
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
| 593 |
-
}
|
| 594 |
-
.nav-button {
|
| 595 |
-
padding: 6px 12px;
|
| 596 |
-
border: 1px solid #ddd;
|
| 597 |
-
border-radius: 20px;
|
| 598 |
-
background-color: white;
|
| 599 |
-
cursor: pointer;
|
| 600 |
-
transition: all 0.3s;
|
| 601 |
-
text-decoration: none;
|
| 602 |
-
color: #333;
|
| 603 |
-
font-size: 13px;
|
| 604 |
-
font-weight: 500;
|
| 605 |
-
}
|
| 606 |
-
.nav-button:hover {
|
| 607 |
-
background-color: #4ECDC4;
|
| 608 |
-
color: white;
|
| 609 |
-
border-color: #4ECDC4;
|
| 610 |
-
}
|
| 611 |
-
</style>
|
| 612 |
-
"""
|
| 613 |
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
all_tables_html += '<strong style="margin-right: 10px;">Quick Navigation:</strong>'
|
| 617 |
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
|
|
|
|
|
|
| 622 |
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
|
|
|
|
|
|
|
|
|
| 626 |
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
icon = '🤝'
|
| 630 |
-
label = 'Shared'
|
| 631 |
-
else:
|
| 632 |
-
icon = '🎯' if entity_type in STANDARD_ENTITIES else '✨'
|
| 633 |
-
label = entity_type
|
| 634 |
-
|
| 635 |
-
all_tables_html += f'<a href="#{entity_type.replace(" ", "_")}" class="nav-button">{icon} {label} ({len(entities_list)})</a>'
|
| 636 |
|
| 637 |
-
|
|
|
|
| 638 |
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
shared_entities_list = entity_groups['SHARED_ENTITIES']
|
| 642 |
-
all_tables_html += f"""
|
| 643 |
-
<div class="entity-section" id="SHARED_ENTITIES">
|
| 644 |
-
<div class="section-header">
|
| 645 |
-
<h3 style="margin: 0; display: flex; align-items: center;">
|
| 646 |
-
<span style="font-size: 24px; margin-right: 10px;">🤝</span>
|
| 647 |
-
Shared Entities
|
| 648 |
-
<span class="entity-count">{len(shared_entities_list)} found</span>
|
| 649 |
-
</h3>
|
| 650 |
-
</div>
|
| 651 |
-
{create_entity_table_html(shared_entities_list, 'SHARED_ENTITIES', '#666666', is_shared=True)}
|
| 652 |
-
</div>
|
| 653 |
-
"""
|
| 654 |
|
| 655 |
-
#
|
| 656 |
-
|
| 657 |
-
if entity_type == 'SHARED_ENTITIES':
|
| 658 |
-
continue
|
| 659 |
-
|
| 660 |
-
colour = entity_colors.get(entity_type.upper(), '#f0f0f0')
|
| 661 |
-
is_standard = entity_type in STANDARD_ENTITIES
|
| 662 |
-
icon = "🎯" if is_standard else "✨"
|
| 663 |
-
type_label = "Common NER" if is_standard else "Custom GLiNER"
|
| 664 |
-
|
| 665 |
-
all_tables_html += f"""
|
| 666 |
-
<div class="entity-section" id="{entity_type.replace(' ', '_')}">
|
| 667 |
-
<div class="section-header">
|
| 668 |
-
<h3 style="margin: 0; display: flex; align-items: center;">
|
| 669 |
-
<span style="font-size: 24px; margin-right: 10px;">{icon}</span>
|
| 670 |
-
{entity_type}
|
| 671 |
-
<span class="entity-count" style="background-color: {colour};">{len(entities_of_type)} found</span>
|
| 672 |
-
</h3>
|
| 673 |
-
<span style="margin-left: auto; color: #666; font-size: 14px;">{type_label}</span>
|
| 674 |
-
</div>
|
| 675 |
-
{create_entity_table_html(entities_of_type, entity_type, colour)}
|
| 676 |
-
</div>
|
| 677 |
-
"""
|
| 678 |
|
| 679 |
-
|
|
|
|
| 680 |
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
"""Create a legend showing entity colours"""
|
| 685 |
-
if not entity_colors:
|
| 686 |
-
return ""
|
| 687 |
-
|
| 688 |
-
html = "<div style='margin: 15px 0; padding: 15px; background-color: #f8f9fa; border-radius: 8px;'>"
|
| 689 |
-
html += "<h4 style='margin: 0 0 15px 0;'>🎨 Entity Type Legend</h4>"
|
| 690 |
-
|
| 691 |
-
if standard_entities:
|
| 692 |
-
html += "<div style='margin-bottom: 15px;'>"
|
| 693 |
-
html += "<h5 style='margin: 0 0 8px 0;'>🎯 Common Entities:</h5>"
|
| 694 |
-
html += "<div style='display: flex; flex-wrap: wrap; gap: 8px;'>"
|
| 695 |
-
for entity_type in standard_entities:
|
| 696 |
-
colour = entity_colors.get(entity_type.upper(), '#ccc')
|
| 697 |
-
html += f"<span style='background-color: {colour}; padding: 4px 8px; border-radius: 15px; color: white; font-weight: bold; font-size: 12px;'>{entity_type}</span>"
|
| 698 |
-
html += "</div></div>"
|
| 699 |
-
|
| 700 |
-
if custom_entities:
|
| 701 |
-
html += "<div>"
|
| 702 |
-
html += "<h5 style='margin: 0 0 8px 0;'>✨ Custom Entities:</h5>"
|
| 703 |
-
html += "<div style='display: flex; flex-wrap: wrap; gap: 8px;'>"
|
| 704 |
-
for entity_type in custom_entities:
|
| 705 |
-
colour = entity_colors.get(entity_type.upper(), '#ccc')
|
| 706 |
-
html += f"<span style='background-color: {colour}; padding: 4px 8px; border-radius: 15px; color: white; font-weight: bold; font-size: 12px;'>{entity_type}</span>"
|
| 707 |
-
html += "</div></div>"
|
| 708 |
-
|
| 709 |
-
html += "</div>"
|
| 710 |
-
return html
|
| 711 |
-
|
| 712 |
-
# Initialize the NER manager
|
| 713 |
-
ner_manager = HybridNERManager()
|
| 714 |
-
|
| 715 |
-
def process_text(text, standard_entities, custom_entities_str, confidence_threshold, selected_model, progress=gr.Progress()):
|
| 716 |
-
"""Main processing function for Gradio interface with progress tracking"""
|
| 717 |
-
if not text.strip():
|
| 718 |
-
return "❌ Please enter some text to analyse", "", "", gr.update(visible=False)
|
| 719 |
-
|
| 720 |
-
progress(0.1, desc="Initialising...")
|
| 721 |
-
|
| 722 |
-
# Parse custom entities
|
| 723 |
-
custom_entities = []
|
| 724 |
-
if custom_entities_str.strip():
|
| 725 |
-
custom_entities = [entity.strip() for entity in custom_entities_str.split(',') if entity.strip()]
|
| 726 |
-
|
| 727 |
-
# Parse common entities
|
| 728 |
-
selected_standard = [entity for entity in standard_entities if entity]
|
| 729 |
-
|
| 730 |
-
if not selected_standard and not custom_entities:
|
| 731 |
-
return "❌ Please select at least one common entity type OR enter custom entity types", "", "", gr.update(visible=False)
|
| 732 |
-
|
| 733 |
-
progress(0.2, desc="Loading models...")
|
| 734 |
-
|
| 735 |
-
all_entities = []
|
| 736 |
-
|
| 737 |
-
# Extract common entities using selected model
|
| 738 |
-
if selected_standard and selected_model:
|
| 739 |
-
progress(0.4, desc="Extracting common entities...")
|
| 740 |
-
standard_entities_results = ner_manager.extract_entities_by_model(text, selected_standard, selected_model, confidence_threshold)
|
| 741 |
-
all_entities.extend(standard_entities_results)
|
| 742 |
-
|
| 743 |
-
# Extract custom entities using GLiNER
|
| 744 |
-
if custom_entities:
|
| 745 |
-
progress(0.6, desc="Extracting custom entities...")
|
| 746 |
-
custom_entity_results = ner_manager.extract_gliner_entities(text, custom_entities, confidence_threshold, is_custom=True)
|
| 747 |
-
all_entities.extend(custom_entity_results)
|
| 748 |
-
|
| 749 |
-
if not all_entities:
|
| 750 |
-
return "❌ No entities found. Try lowering the confidence threshold or using different entity types.", "", "", gr.update(visible=False)
|
| 751 |
-
|
| 752 |
-
progress(0.8, desc="Processing results...")
|
| 753 |
|
| 754 |
-
#
|
| 755 |
-
|
|
|
|
| 756 |
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
highlighted_html = create_highlighted_html(text, all_entities, entity_colors)
|
| 760 |
-
results_html = create_all_entity_tables(all_entities, entity_colors)
|
| 761 |
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
total_entities = len(all_entities)
|
| 766 |
-
shared_entities = find_overlapping_entities(all_entities)
|
| 767 |
-
final_count = len(shared_entities)
|
| 768 |
-
shared_count = sum(1 for e in shared_entities if e.get('is_shared', False))
|
| 769 |
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
- **Final entities displayed:** {final_count}
|
| 774 |
-
- **Shared entities:** {shared_count}
|
| 775 |
-
- **Average confidence:** {sum(e.get('confidence', 0) for e in all_entities) / total_entities:.3f}
|
| 776 |
-
"""
|
| 777 |
-
|
| 778 |
-
progress(1.0, desc="Complete!")
|
| 779 |
|
| 780 |
-
return
|
| 781 |
|
| 782 |
-
# Create Gradio interface
|
| 783 |
def create_interface():
|
| 784 |
-
with gr.Blocks(title="
|
| 785 |
gr.Markdown("""
|
| 786 |
-
#
|
| 787 |
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
1.
|
| 793 |
-
2.
|
| 794 |
-
3.
|
| 795 |
-
4.
|
| 796 |
-
5. **⚙️ Adjust confidence threshold**
|
| 797 |
-
6. **🔍 Click "Analyse Text"** to see results
|
| 798 |
-
(NB: common/custom entities which overlap are shown with split-colour highlighting)
|
| 799 |
-
7. **🔄 Refresh the page** to try again with new text
|
| 800 |
-
""")
|
| 801 |
-
# Add tip box
|
| 802 |
-
gr.HTML("""
|
| 803 |
-
<div style="background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 8px; padding: 12px; margin: 15px 0;">
|
| 804 |
-
<strong style="color: #856404;">💡 Top tip:</strong> No model is perfect - all can miss and/or incorrectly identify entity types.
|
| 805 |
-
</div>
|
| 806 |
""")
|
| 807 |
|
| 808 |
with gr.Row():
|
| 809 |
-
with gr.Column(scale=
|
| 810 |
text_input = gr.Textbox(
|
| 811 |
-
label="
|
| 812 |
-
placeholder="Enter
|
| 813 |
-
lines=
|
| 814 |
max_lines=20
|
| 815 |
)
|
| 816 |
|
| 817 |
with gr.Column(scale=1):
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
|
| 821 |
-
|
| 822 |
-
|
| 823 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 824 |
)
|
| 825 |
-
|
| 826 |
-
# Add confidence threshold explanation
|
| 827 |
-
gr.HTML("""
|
| 828 |
-
<details style="margin: 10px 0; padding: 10px; background-color: #f8f9fa; border-radius: 8px; border: 1px solid #ddd;">
|
| 829 |
-
<summary style="cursor: pointer; font-weight: bold; padding: 5px; color: #1976d2;">
|
| 830 |
-
ℹ️ Understanding the Confidence Threshold
|
| 831 |
-
</summary>
|
| 832 |
-
<div style="margin-top: 10px; padding: 10px; font-size: 14px;">
|
| 833 |
-
<p style="margin: 0 0 10px 0;">The <strong>confidence threshold</strong> controls how certain the model needs to be before identifying an entity:</p>
|
| 834 |
-
<ul style="margin: 0; padding-left: 20px;">
|
| 835 |
-
<li><strong>Lower values (0.1-0.3):</strong> More entities detected, but may include false positives</li>
|
| 836 |
-
<li><strong>Medium values (0.4-0.6):</strong> Balanced detection with moderate confidence</li>
|
| 837 |
-
<li><strong>Higher values (0.7-0.9):</strong> Only highly confident entities detected, may miss some valid entities</li>
|
| 838 |
-
</ul>
|
| 839 |
-
<p style="margin: 10px 0 0 0; font-style: italic;"><strong>Top Tip:</strong> Start with 0.3 for comprehensive detection, then adjust based on your needs.</p>
|
| 840 |
-
</div>
|
| 841 |
-
</details>
|
| 842 |
-
""")
|
| 843 |
|
| 844 |
with gr.Row():
|
| 845 |
-
|
| 846 |
-
|
| 847 |
-
|
| 848 |
-
# Model selector
|
| 849 |
-
model_dropdown = gr.Dropdown(
|
| 850 |
-
choices=ner_manager.model_names,
|
| 851 |
-
value=ner_manager.model_names[0],
|
| 852 |
-
label="Select NER Model for Common Entities"
|
| 853 |
-
)
|
| 854 |
-
|
| 855 |
-
# Common entities with select all functionality
|
| 856 |
-
standard_entities = gr.CheckboxGroup(
|
| 857 |
-
choices=STANDARD_ENTITIES,
|
| 858 |
-
value=['PERSON', 'ORGANIZATION', 'LOCATION', 'MISCELLANEOUS'], # Default selection with full names
|
| 859 |
-
label="Select Common Entities"
|
| 860 |
-
)
|
| 861 |
-
|
| 862 |
-
# Select/Deselect All button
|
| 863 |
-
with gr.Row():
|
| 864 |
-
select_all_btn = gr.Button("🔘 Deselect All", size="sm")
|
| 865 |
-
|
| 866 |
-
# Function for select/deselect all
|
| 867 |
-
def toggle_all_entities(current_selection):
|
| 868 |
-
if len(current_selection) > 0:
|
| 869 |
-
# If any are selected, deselect all
|
| 870 |
-
return [], "☑️ Select All"
|
| 871 |
-
else:
|
| 872 |
-
# If none selected, select all
|
| 873 |
-
return STANDARD_ENTITIES, "🔘 Deselect All"
|
| 874 |
-
|
| 875 |
-
select_all_btn.click(
|
| 876 |
-
fn=toggle_all_entities,
|
| 877 |
-
inputs=[standard_entities],
|
| 878 |
-
outputs=[standard_entities, select_all_btn]
|
| 879 |
-
)
|
| 880 |
-
|
| 881 |
-
with gr.Column():
|
| 882 |
-
gr.Markdown("### ✨ Custom Entity Types - Powered by GLiNER")
|
| 883 |
-
custom_entities = gr.Textbox(
|
| 884 |
-
label="Custom Entities (comma-separated)",
|
| 885 |
-
placeholder="e.g. relationships, civilian occupations, emotions",
|
| 886 |
-
lines=3
|
| 887 |
-
)
|
| 888 |
-
gr.Markdown("""
|
| 889 |
-
**Examples:**
|
| 890 |
-
- relationships, occupations, skills
|
| 891 |
-
- emotions, actions, objects
|
| 892 |
-
- medical conditions, treatments
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
""")
|
| 897 |
-
|
| 898 |
-
# Add glossary (alphabetically ordered)
|
| 899 |
-
gr.HTML("""
|
| 900 |
-
<details style="margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 8px; border: 1px solid #ddd;">
|
| 901 |
-
<summary style="cursor: pointer; font-weight: bold; padding: 5px; color: #1976d2;">
|
| 902 |
-
ℹ️ Entity Type Definitions
|
| 903 |
-
</summary>
|
| 904 |
-
<div style="margin-top: 10px; padding: 10px;">
|
| 905 |
-
<dl style="margin: 0; font-size: 14px;">
|
| 906 |
-
<div style="margin-bottom: 8px;">
|
| 907 |
-
<dt style="font-weight: bold; display: inline; color: #FF6B6B;">Date (DATE):</dt>
|
| 908 |
-
<dd style="display: inline; margin-left: 5px;">Absolute or relative dates or periods</dd>
|
| 909 |
-
</div>
|
| 910 |
-
<div style="margin-bottom: 8px;">
|
| 911 |
-
<dt style="font-weight: bold; display: inline; color: #4ECDC4;">Event (EVENT):</dt>
|
| 912 |
-
<dd style="display: inline; margin-left: 5px;">Named hurricanes, battles, wars, sports events, etc.</dd>
|
| 913 |
-
</div>
|
| 914 |
-
<div style="margin-bottom: 8px;">
|
| 915 |
-
<dt style="font-weight: bold; display: inline; color: #45B7D1;">Facility (FAC):</dt>
|
| 916 |
-
<dd style="display: inline; margin-left: 5px;">Buildings, airports, highways, bridges, etc.</dd>
|
| 917 |
-
</div>
|
| 918 |
-
<div style="margin-bottom: 8px;">
|
| 919 |
-
<dt style="font-weight: bold; display: inline; color: #F9CA24;">Geopolitical Entity (GPE):</dt>
|
| 920 |
-
<dd style="display: inline; margin-left: 5px;">Countries, cities, states</dd>
|
| 921 |
-
</div>
|
| 922 |
-
<div style="margin-bottom: 8px;">
|
| 923 |
-
<dt style="font-weight: bold; display: inline; color: #6C5CE7;">Language (LANG):</dt>
|
| 924 |
-
<dd style="display: inline; margin-left: 5px;">Any named language</dd>
|
| 925 |
-
</div>
|
| 926 |
-
<div style="margin-bottom: 8px;">
|
| 927 |
-
<dt style="font-weight: bold; display: inline; color: #A0E7E5;">Location (LOC):</dt>
|
| 928 |
-
<dd style="display: inline; margin-left: 5px;">Non-GPE locations - Mountain ranges, bodies of water</dd>
|
| 929 |
-
</div>
|
| 930 |
-
<div style="margin-bottom: 8px;">
|
| 931 |
-
<dt style="font-weight: bold; display: inline; color: #FD79A8;">Miscellaneous (MISC):</dt>
|
| 932 |
-
<dd style="display: inline; margin-left: 5px;">Entities that don't fit elsewhere</dd>
|
| 933 |
-
</div>
|
| 934 |
-
<div style="margin-bottom: 8px;">
|
| 935 |
-
<dt style="font-weight: bold; display: inline; color: #8E8E93;">Nationalities/Groups (NORP):</dt>
|
| 936 |
-
<dd style="display: inline; margin-left: 5px;">Nationalities or religious or political groups</dd>
|
| 937 |
-
</div>
|
| 938 |
-
<div style="margin-bottom: 8px;">
|
| 939 |
-
<dt style="font-weight: bold; display: inline; color: #55A3FF;">Organization (ORG):</dt>
|
| 940 |
-
<dd style="display: inline; margin-left: 5px;">Companies, agencies, institutions, etc.</dd>
|
| 941 |
-
</div>
|
| 942 |
-
<div style="margin-bottom: 8px;">
|
| 943 |
-
<dt style="font-weight: bold; display: inline; color: #00B894;">Person (PER):</dt>
|
| 944 |
-
<dd style="display: inline; margin-left: 5px;">People, including fictional characters</dd>
|
| 945 |
-
</div>
|
| 946 |
-
<div style="margin-bottom: 8px;">
|
| 947 |
-
<dt style="font-weight: bold; display: inline; color: #E17055;">Product (PRODUCT):</dt>
|
| 948 |
-
<dd style="display: inline; margin-left: 5px;">Objects, vehicles, foods, etc. (Not services)</dd>
|
| 949 |
-
</div>
|
| 950 |
-
<div style="margin-bottom: 8px;">
|
| 951 |
-
<dt style="font-weight: bold; display: inline; color: #DDA0DD;">Work of Art (Work of Art):</dt>
|
| 952 |
-
<dd style="display: inline; margin-left: 5px;">Titles of books, songs, movies, paintings, etc.</dd>
|
| 953 |
-
</div>
|
| 954 |
-
</dl>
|
| 955 |
-
</div>
|
| 956 |
-
</details>
|
| 957 |
-
""")
|
| 958 |
-
|
| 959 |
-
analyse_btn = gr.Button("🔍 Analyse Text", variant="primary", size="lg")
|
| 960 |
|
| 961 |
-
# Add horizontal line separator after the button
|
| 962 |
-
gr.HTML("<hr style='margin-top: 20px; margin-bottom: 15px;'>")
|
| 963 |
-
|
| 964 |
-
# Output sections
|
| 965 |
with gr.Row():
|
| 966 |
-
|
| 967 |
|
| 968 |
with gr.Row():
|
| 969 |
-
highlighted_output = gr.HTML(label="Highlighted
|
| 970 |
-
|
| 971 |
-
# Results section (initially hidden)
|
| 972 |
-
with gr.Row(visible=False) as results_row:
|
| 973 |
-
with gr.Column():
|
| 974 |
-
gr.Markdown("### 📋 Detailed Results")
|
| 975 |
-
results_output = gr.HTML(label="Entity Results")
|
| 976 |
-
|
| 977 |
-
# Connect the button to the processing function
|
| 978 |
-
analyse_btn.click(
|
| 979 |
-
fn=process_text,
|
| 980 |
-
inputs=[
|
| 981 |
-
text_input,
|
| 982 |
-
standard_entities,
|
| 983 |
-
custom_entities,
|
| 984 |
-
confidence_threshold,
|
| 985 |
-
model_dropdown
|
| 986 |
-
],
|
| 987 |
-
outputs=[summary_output, highlighted_output, results_output, results_row]
|
| 988 |
-
)
|
| 989 |
-
|
| 990 |
-
# Updated examples text with reduced spacing
|
| 991 |
-
with gr.Column():
|
| 992 |
-
gr.Markdown("""
|
| 993 |
-
### 💡 No example text to test? No problem!
|
| 994 |
-
Simply click on one of the examples provided below, and the fields will be populated for you.
|
| 995 |
-
""", elem_id="examples-heading")
|
| 996 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 997 |
gr.Examples(
|
| 998 |
examples=[
|
| 999 |
[
|
| 1000 |
-
"
|
| 1001 |
-
|
| 1002 |
-
|
| 1003 |
-
|
| 1004 |
-
|
| 1005 |
],
|
| 1006 |
[
|
| 1007 |
-
"
|
| 1008 |
-
|
| 1009 |
-
|
| 1010 |
-
|
| 1011 |
-
|
| 1012 |
],
|
| 1013 |
[
|
| 1014 |
-
"
|
| 1015 |
-
|
| 1016 |
-
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
]
|
| 1020 |
],
|
| 1021 |
-
inputs=[
|
| 1022 |
-
|
| 1023 |
-
standard_entities,
|
| 1024 |
-
custom_entities,
|
| 1025 |
-
confidence_threshold,
|
| 1026 |
-
model_dropdown
|
| 1027 |
-
],
|
| 1028 |
-
label="Examples"
|
| 1029 |
)
|
| 1030 |
|
| 1031 |
-
#
|
| 1032 |
-
|
| 1033 |
-
|
| 1034 |
-
|
| 1035 |
-
|
| 1036 |
-
|
| 1037 |
-
|
| 1038 |
-
|
| 1039 |
-
|
| 1040 |
-
|
| 1041 |
-
|
| 1042 |
-
|
| 1043 |
-
|
| 1044 |
-
|
| 1045 |
-
|
| 1046 |
-
|
| 1047 |
-
|
| 1048 |
-
|
| 1049 |
-
|
| 1050 |
-
|
| 1051 |
-
|
| 1052 |
-
|
| 1053 |
-
|
| 1054 |
-
|
| 1055 |
-
|
| 1056 |
-
|
| 1057 |
-
|
| 1058 |
-
|
| 1059 |
-
|
| 1060 |
-
|
| 1061 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1062 |
""")
|
| 1063 |
|
| 1064 |
-
#
|
| 1065 |
gr.HTML("""
|
| 1066 |
-
<
|
| 1067 |
-
|
| 1068 |
-
|
| 1069 |
-
|
| 1070 |
-
|
| 1071 |
-
<li><strong>flair_ner-large:</strong>
|
| 1072 |
-
<a href="https://huggingface.co/flair/ner-english-large" target="_blank" style="color: #1976d2;">
|
| 1073 |
-
Flair NER English Large Model ↗
|
| 1074 |
-
</a>
|
| 1075 |
-
</li>
|
| 1076 |
-
<li><strong>spacy_en_core_web_trf:</strong>
|
| 1077 |
-
<a href="https://spacy.io/models/en#en_core_web_trf" target="_blank" style="color: #1976d2;">
|
| 1078 |
-
spaCy English Transformer Model ↗
|
| 1079 |
-
</a>
|
| 1080 |
-
</li>
|
| 1081 |
-
<li><strong>flair_ner-ontonotes-large:</strong>
|
| 1082 |
-
<a href="https://huggingface.co/flair/ner-english-ontonotes-large" target="_blank" style="color: #1976d2;">
|
| 1083 |
-
Flair OntoNotes Large Model ↗
|
| 1084 |
-
</a>
|
| 1085 |
-
</li>
|
| 1086 |
-
<li><strong>gliner_knowledgator/modern-gliner-bi-large-v1.0:</strong>
|
| 1087 |
-
<a href="https://github.com/urchade/GLiNER/blob/main/README_Extended.md" target="_blank" style="color: #1976d2;">
|
| 1088 |
-
GLiNER Extended Documentation ↗
|
| 1089 |
-
</a>
|
| 1090 |
-
</li>
|
| 1091 |
-
</ul>
|
| 1092 |
</div>
|
| 1093 |
-
|
| 1094 |
-
|
| 1095 |
-
<hr style="margin-top: 40px; margin-bottom: 20px;">
|
| 1096 |
-
<div style="background-color: #f8f9fa; padding: 20px; border-radius: 8px; margin-top: 20px; text-align: center;">
|
| 1097 |
-
<p style="font-size: 14px; line-height: 1.8; margin: 0;">
|
| 1098 |
-
This <strong>NER Explorer Tool</strong> was created as part of the
|
| 1099 |
-
<a href="https://digitalscholarship.web.ox.ac.uk/" target="_blank" style="color: #1976d2;">
|
| 1100 |
-
Digital Scholarship at Oxford (DiSc)
|
| 1101 |
-
</a>
|
| 1102 |
-
funded research project:<br>
|
| 1103 |
-
<em>Extracting Keywords from Crowdsourced Collections</em>.
|
| 1104 |
-
</p><br><br>
|
| 1105 |
-
<p style="font-size: 14px; line-height: 1.8; margin: 0;">
|
| 1106 |
-
The code for this tool was built with the aid of Claude Opus 4.
|
| 1107 |
-
</p>
|
| 1108 |
-
</div>
|
| 1109 |
-
""")
|
| 1110 |
-
|
| 1111 |
return demo
|
| 1112 |
|
| 1113 |
if __name__ == "__main__":
|
|
|
|
| 1 |
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
import re
|
| 3 |
+
import pandas as pd
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
+
def parse_keywords_dict(keywords_text):
|
| 6 |
+
"""Parse the keyword dictionary from text input"""
|
| 7 |
+
keywords_dict = {}
|
| 8 |
+
|
| 9 |
+
if not keywords_text.strip():
|
| 10 |
+
return keywords_dict
|
| 11 |
+
|
| 12 |
+
lines = keywords_text.strip().split('\n')
|
| 13 |
+
for line in lines:
|
| 14 |
+
line = line.strip()
|
| 15 |
+
if not line or '|' not in line:
|
| 16 |
+
continue
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
try:
|
| 19 |
+
primary, secondary = line.split('|', 1)
|
| 20 |
+
primary = primary.strip()
|
| 21 |
+
secondary_list = [keyword.strip() for keyword in secondary.split(';') if keyword.strip()]
|
| 22 |
+
keywords_dict[primary] = secondary_list
|
| 23 |
+
except:
|
| 24 |
+
continue
|
| 25 |
+
|
| 26 |
+
return keywords_dict
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
+
def find_keywords(story, keywords_dict):
|
| 29 |
+
"""Find keywords in the story text"""
|
| 30 |
+
if not story or not isinstance(story, str):
|
| 31 |
+
return ''
|
| 32 |
+
|
| 33 |
+
found_keywords = set()
|
| 34 |
+
|
| 35 |
+
# Search for each primary keyword and its synonyms
|
| 36 |
+
for primary_keyword, synonyms in keywords_dict.items():
|
| 37 |
+
# Special handling for 'US' to avoid matching 'us'
|
| 38 |
+
if primary_keyword.upper() == "US":
|
| 39 |
+
if ' US ' in story or story.startswith('US ') or story.endswith(' US'):
|
| 40 |
+
found_keywords.update(synonyms)
|
| 41 |
else:
|
| 42 |
+
pattern = r'\b' + re.escape(primary_keyword) + r'\b'
|
| 43 |
+
if re.search(pattern, story, re.IGNORECASE):
|
| 44 |
+
found_keywords.update(synonyms)
|
| 45 |
+
|
| 46 |
+
# Check each secondary keyword independently
|
| 47 |
+
for synonym in synonyms:
|
| 48 |
+
if synonym.upper() == "US":
|
| 49 |
+
if ' US ' in story or story.startswith('US ') or story.endswith(' US'):
|
| 50 |
+
found_keywords.add(synonym)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
else:
|
| 52 |
+
if re.search(r'\b' + re.escape(synonym) + r'\b', story, re.IGNORECASE):
|
| 53 |
+
found_keywords.add(synonym)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
+
return '; '.join(sorted(found_keywords))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
+
def highlight_keywords_in_text(text, keywords_list):
|
| 58 |
+
"""Create HTML with highlighted keywords"""
|
| 59 |
+
if not keywords_list:
|
| 60 |
+
return text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
+
highlighted_text = text
|
| 63 |
+
colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#F9CA24', '#6C5CE7', '#A0E7E5', '#FD79A8', '#55A3FF', '#00B894', '#E17055']
|
|
|
|
| 64 |
|
| 65 |
+
for i, keyword in enumerate(keywords_list):
|
| 66 |
+
if keyword:
|
| 67 |
+
color = colors[i % len(colors)]
|
| 68 |
+
pattern = r'\b' + re.escape(keyword) + r'\b'
|
| 69 |
+
replacement = f'<span style="background-color: {color}; padding: 2px 4px; border-radius: 3px; color: white; font-weight: bold;">{keyword}</span>'
|
| 70 |
+
highlighted_text = re.sub(pattern, replacement, highlighted_text, flags=re.IGNORECASE)
|
| 71 |
|
| 72 |
+
return highlighted_text
|
| 73 |
+
|
| 74 |
+
def process_text(input_text, keywords_text):
|
| 75 |
+
"""Main processing function"""
|
| 76 |
+
if not input_text.strip():
|
| 77 |
+
return "Please enter some text to analyze", "", "No keywords found"
|
| 78 |
|
| 79 |
+
if not keywords_text.strip():
|
| 80 |
+
return "Please enter keyword mappings", "", "No keyword dictionary provided"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
+
# Parse keywords dictionary
|
| 83 |
+
keywords_dict = parse_keywords_dict(keywords_text)
|
| 84 |
|
| 85 |
+
if not keywords_dict:
|
| 86 |
+
return "Invalid keyword format. Please check your keyword dictionary format.", "", "Error parsing keywords"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
+
# Find keywords in the text
|
| 89 |
+
found_keywords_str = find_keywords(input_text, keywords_dict)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
+
if not found_keywords_str:
|
| 92 |
+
return f"No keywords found in the text.\n\nKeyword dictionary loaded: {len(keywords_dict)} primary keywords", input_text, "No matches found"
|
| 93 |
|
| 94 |
+
# Create highlighted version
|
| 95 |
+
keywords_list = found_keywords_str.split('; ')
|
| 96 |
+
highlighted_html = highlight_keywords_in_text(input_text, keywords_list)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
+
# Create results summary
|
| 99 |
+
results_summary = f"""
|
| 100 |
+
## Results Summary
|
| 101 |
|
| 102 |
+
**Keywords Found:** {len(keywords_list)}
|
| 103 |
+
**Matched Keywords:** {found_keywords_str}
|
|
|
|
|
|
|
| 104 |
|
| 105 |
+
**Keyword Dictionary Stats:**
|
| 106 |
+
- Primary keywords loaded: {len(keywords_dict)}
|
| 107 |
+
- Total searchable terms: {sum(len(synonyms) + 1 for synonyms in keywords_dict.values())}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
|
| 109 |
+
**Copy this result to your spreadsheet:**
|
| 110 |
+
{found_keywords_str}
|
| 111 |
+
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
+
return results_summary, highlighted_html, found_keywords_str
|
| 114 |
|
| 115 |
+
# Create the Gradio interface
|
| 116 |
def create_interface():
|
| 117 |
+
with gr.Blocks(title="Keyword Tagging Tool", theme=gr.themes.Soft()) as demo:
|
| 118 |
gr.Markdown("""
|
| 119 |
+
# Keyword Tagging Tool
|
| 120 |
|
| 121 |
+
This tool matches text against a controlled vocabulary and returns all associated keywords.
|
| 122 |
+
Based on the keyword matching logic used for digital humanities research.
|
| 123 |
+
|
| 124 |
+
## How to use:
|
| 125 |
+
1. **Enter your text** in the left panel
|
| 126 |
+
2. **Define your keyword dictionary** in the right panel using the format: `Primary Keyword | synonym1; synonym2; synonym3`
|
| 127 |
+
3. **Click "Find Keywords"** to see results
|
| 128 |
+
4. **Copy the results** to paste into your spreadsheet
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
""")
|
| 130 |
|
| 131 |
with gr.Row():
|
| 132 |
+
with gr.Column(scale=1):
|
| 133 |
text_input = gr.Textbox(
|
| 134 |
+
label="Text to Analyze",
|
| 135 |
+
placeholder="Enter the text you want to tag with keywords...",
|
| 136 |
+
lines=15,
|
| 137 |
max_lines=20
|
| 138 |
)
|
| 139 |
|
| 140 |
with gr.Column(scale=1):
|
| 141 |
+
keywords_input = gr.Textbox(
|
| 142 |
+
label="Keyword Dictionary",
|
| 143 |
+
placeholder="""Enter one mapping per line:
|
| 144 |
+
Primary Keyword | synonym1; synonym2; synonym3
|
| 145 |
+
|
| 146 |
+
Example:
|
| 147 |
+
Prisoner of War | Prisoner of War; POW; POWs
|
| 148 |
+
United States | United States; USA; US; America
|
| 149 |
+
""",
|
| 150 |
+
lines=15,
|
| 151 |
+
max_lines=20
|
| 152 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
with gr.Row():
|
| 155 |
+
find_btn = gr.Button("Find Keywords", variant="primary", size="lg")
|
| 156 |
+
clear_btn = gr.Button("Clear All", size="lg")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
with gr.Row():
|
| 159 |
+
results_output = gr.Markdown(label="Results Summary")
|
| 160 |
|
| 161 |
with gr.Row():
|
| 162 |
+
highlighted_output = gr.HTML(label="Text with Highlighted Keywords")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
|
| 164 |
+
with gr.Row():
|
| 165 |
+
copy_output = gr.Textbox(
|
| 166 |
+
label="Keywords for Spreadsheet (copy this text)",
|
| 167 |
+
lines=3,
|
| 168 |
+
max_lines=5
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
# Examples section
|
| 172 |
+
gr.Markdown("### Examples")
|
| 173 |
gr.Examples(
|
| 174 |
examples=[
|
| 175 |
[
|
| 176 |
+
"During World War II, many prisoners of war were held in camps across Europe. The Geneva Convention established rules for POW treatment. American soldiers and British troops were among those captured.",
|
| 177 |
+
"""Prisoner of War | Prisoner of War; POW; POWs; prisoner of war
|
| 178 |
+
World War II | World War II; WWII; Second World War
|
| 179 |
+
United States | United States; USA; US; America; American
|
| 180 |
+
United Kingdom | United Kingdom; UK; Britain; British"""
|
| 181 |
],
|
| 182 |
[
|
| 183 |
+
"The University of Oxford is located in Oxford, England. Students from around the world study at this prestigious institution. The university has many colleges including Christ Church and Magdalen College.",
|
| 184 |
+
"""University | University; university; institution; college
|
| 185 |
+
Oxford | Oxford; oxford
|
| 186 |
+
England | England; england; English
|
| 187 |
+
Student | Student; student; students; pupils"""
|
| 188 |
],
|
| 189 |
[
|
| 190 |
+
"Shakespeare wrote many famous plays including Hamlet, Romeo and Juliet, and Macbeth. These works are performed in theatres worldwide and studied in schools.",
|
| 191 |
+
"""William Shakespeare | Shakespeare; william shakespeare; playwright
|
| 192 |
+
Theatre | Theatre; theater; stage; performance
|
| 193 |
+
Play | Play; plays; drama; theatrical work
|
| 194 |
+
School | School; schools; education; academic"""
|
| 195 |
]
|
| 196 |
],
|
| 197 |
+
inputs=[text_input, keywords_input],
|
| 198 |
+
label="Click an example to try it out"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
)
|
| 200 |
|
| 201 |
+
# Button functions
|
| 202 |
+
find_btn.click(
|
| 203 |
+
fn=process_text,
|
| 204 |
+
inputs=[text_input, keywords_input],
|
| 205 |
+
outputs=[results_output, highlighted_output, copy_output]
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
def clear_all():
|
| 209 |
+
return "", "", "", "", ""
|
| 210 |
+
|
| 211 |
+
clear_btn.click(
|
| 212 |
+
fn=clear_all,
|
| 213 |
+
outputs=[text_input, keywords_input, results_output, highlighted_output, copy_output]
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
# Instructions
|
| 217 |
+
gr.Markdown("""
|
| 218 |
+
## Format Guide
|
| 219 |
+
|
| 220 |
+
**Keyword Dictionary Format:**
|
| 221 |
+
- One primary keyword per line
|
| 222 |
+
- Use the pipe symbol `|` to separate primary keyword from synonyms
|
| 223 |
+
- Use semicolons `;` to separate multiple synonyms
|
| 224 |
+
- Case-insensitive matching (except for special cases like "US")
|
| 225 |
+
|
| 226 |
+
**Special Handling:**
|
| 227 |
+
- "US" is matched exactly to avoid confusion with the word "us"
|
| 228 |
+
- Word boundaries are respected (prevents partial matches)
|
| 229 |
+
- Results are alphabetized and deduplicated
|
| 230 |
+
|
| 231 |
+
**Example Dictionary Entry:**
|
| 232 |
+
```
|
| 233 |
+
Prisoner of War | Prisoner of War; POW; POWs; prisoner of war
|
| 234 |
+
```
|
| 235 |
+
|
| 236 |
+
This will find any occurrence of "Prisoner of War", "POW", "POWs", or "prisoner of war" in your text and return all the associated terms.
|
| 237 |
""")
|
| 238 |
|
| 239 |
+
# Footer
|
| 240 |
gr.HTML("""
|
| 241 |
+
<div style="margin-top: 40px; padding: 20px; background-color: #f8f9fa; border-radius: 8px; text-align: center;">
|
| 242 |
+
<p style="margin: 0; color: #666;">
|
| 243 |
+
Created for digital humanities keyword tagging workflows.
|
| 244 |
+
Based on controlled vocabulary matching principles.
|
| 245 |
+
</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
</div>
|
| 247 |
+
""")
|
| 248 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
return demo
|
| 250 |
|
| 251 |
if __name__ == "__main__":
|