SorrelC commited on
Commit
72f65df
·
verified ·
1 Parent(s): 38ec28d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +189 -1051
app.py CHANGED
@@ -1,1113 +1,251 @@
1
  import gradio as gr
2
- import torch
3
- from gliner import GLiNER
4
- import pandas as pd
5
- import warnings
6
- import random
7
  import re
8
- import time
9
- warnings.filterwarnings('ignore')
10
-
11
- # Common NER entity types (using full names)
12
- STANDARD_ENTITIES = [
13
- 'DATE', 'EVENT', 'FACILITY', 'GEOPOLITICAL ENTITY', 'LANGUAGE', 'LOCATION',
14
- 'MISCELLANEOUS', 'NATIONALITIES/GROUPS', 'ORGANIZATION', 'PERSON', 'PRODUCT', 'WORK OF ART'
15
- ]
16
-
17
- # Colour schemes (updated to match full names)
18
- STANDARD_COLORS = {
19
- 'DATE': '#FF6B6B', # Red
20
- 'EVENT': '#4ECDC4', # Teal
21
- 'FACILITY': '#45B7D1', # Blue
22
- 'GEOPOLITICAL ENTITY': '#F9CA24', # Yellow
23
- 'LANGUAGE': '#6C5CE7', # Purple
24
- 'LOCATION': '#A0E7E5', # Light Cyan
25
- 'MISCELLANEOUS': '#FD79A8', # Pink
26
- 'NATIONALITIES/GROUPS': '#8E8E93', # Grey
27
- 'ORGANIZATION': '#55A3FF', # Light Blue
28
- 'PERSON': '#00B894', # Green
29
- 'PRODUCT': '#E17055', # Orange-Red
30
- 'WORK OF ART': '#DDA0DD' # Plum
31
- }
32
-
33
- # Entity definitions for glossary (alphabetically ordered with full name (abbreviation) format)
34
- ENTITY_DEFINITIONS = {
35
- 'DATE': 'Date (DATE): Absolute or relative dates or periods',
36
- 'EVENT': 'Event (EVENT): Named hurricanes, battles, wars, sports events, etc.',
37
- 'FACILITY': 'Facility (FAC): Buildings, airports, highways, bridges, etc.',
38
- 'GEOPOLITICAL ENTITY': 'Geopolitical Entity (GPE): Countries, cities, states',
39
- 'LANGUAGE': 'Language (LANG): Any named language',
40
- 'LOCATION': 'Location (LOC): Non-GPE locations - Mountain ranges, bodies of water',
41
- 'MISCELLANEOUS': 'Miscellaneous (MISC): Entities that don\'t fit elsewhere',
42
- 'NATIONALITIES/GROUPS': 'Nationalities/Groups (NORP): Nationalities or religious or political groups',
43
- 'ORGANIZATION': 'Organization (ORG): Companies, agencies, institutions, etc.',
44
- 'PERSON': 'Person (PER): People, including fictional characters',
45
- 'PRODUCT': 'Product (PRODUCT): Objects, vehicles, foods, etc. (Not services)',
46
- 'WORK OF ART': 'Work of Art (Work of Art): Titles of books, songs, movies, paintings, etc.'
47
- }
48
-
49
- # Additional colours for custom entities
50
- CUSTOM_COLOR_PALETTE = [
51
- '#FF9F43', '#10AC84', '#EE5A24', '#0FBC89', '#5F27CD',
52
- '#FF3838', '#2F3640', '#3742FA', '#2ED573', '#FFA502',
53
- '#FF6348', '#1E90FF', '#FF1493', '#32CD32', '#FFD700',
54
- '#FF4500', '#DA70D6', '#00CED1', '#FF69B4', '#7B68EE'
55
- ]
56
-
57
- class HybridNERManager:
58
- def __init__(self):
59
- self.gliner_model = None
60
- self.spacy_model = None
61
- self.flair_models = {}
62
- self.all_entity_colors = {}
63
- self.model_names = [
64
- 'flair_ner-large',
65
- 'spacy_en_core_web_trf',
66
- 'flair_ner-ontonotes-large',
67
- 'gliner_knowledgator/modern-gliner-bi-large-v1.0'
68
- ]
69
- # Mapping from full names to abbreviations for model compatibility
70
- self.entity_mapping = {
71
- 'DATE': 'DATE',
72
- 'EVENT': 'EVENT',
73
- 'FACILITY': 'FAC',
74
- 'GEOPOLITICAL ENTITY': 'GPE',
75
- 'LANGUAGE': 'LANG',
76
- 'LOCATION': 'LOC',
77
- 'MISCELLANEOUS': 'MISC',
78
- 'NATIONALITIES/GROUPS': 'NORP',
79
- 'ORGANIZATION': 'ORG',
80
- 'PERSON': 'PER',
81
- 'PRODUCT': 'PRODUCT',
82
- 'WORK OF ART': 'Work of Art'
83
- }
84
- # Reverse mapping for display
85
- self.abbrev_to_full = {v: k for k, v in self.entity_mapping.items()}
86
-
87
- def load_model(self, model_name):
88
- """Load the specified model"""
89
- try:
90
- if 'spacy' in model_name:
91
- return self.load_spacy_model()
92
- elif 'flair' in model_name:
93
- return self.load_flair_model(model_name)
94
- elif 'gliner' in model_name:
95
- return self.load_gliner_model()
96
- except Exception as e:
97
- print(f"Error loading {model_name}: {str(e)}")
98
- return None
99
-
100
- def load_spacy_model(self):
101
- """Load spaCy model for common NER"""
102
- if self.spacy_model is None:
103
- try:
104
- import spacy
105
- try:
106
- # Try transformer model first, fallback to small model
107
- self.spacy_model = spacy.load("en_core_web_trf")
108
- print("✓ spaCy transformer model loaded successfully")
109
- except OSError:
110
- try:
111
- self.spacy_model = spacy.load("en_core_web_sm")
112
- print("✓ spaCy common model loaded successfully")
113
- except OSError:
114
- print("spaCy model not found. Using GLiNER for all entity types.")
115
- return None
116
- except Exception as e:
117
- print(f"Error loading spaCy model: {str(e)}")
118
- return None
119
- return self.spacy_model
120
-
121
- def load_flair_model(self, model_name):
122
- """Load Flair models"""
123
- if model_name not in self.flair_models:
124
- try:
125
- from flair.models import SequenceTagger
126
- if 'ontonotes' in model_name:
127
- model = SequenceTagger.load("flair/ner-english-ontonotes-large")
128
- print("✓ Flair OntoNotes model loaded successfully")
129
- else:
130
- model = SequenceTagger.load("flair/ner-english-large")
131
- print("✓ Flair large model loaded successfully")
132
- self.flair_models[model_name] = model
133
- except Exception as e:
134
- print(f"Error loading {model_name}: {str(e)}")
135
- # Fallback to GLiNER
136
- return self.load_gliner_model()
137
- return self.flair_models[model_name]
138
-
139
- def load_gliner_model(self):
140
- """Load GLiNER model for custom entities"""
141
- if self.gliner_model is None:
142
- try:
143
- # Try the modern GLiNER model first, fallback to stable model
144
- self.gliner_model = GLiNER.from_pretrained("knowledgator/gliner-bi-large-v1.0")
145
- print("✓ GLiNER knowledgator model loaded successfully")
146
- except Exception as e:
147
- print(f"Primary GLiNER model failed: {str(e)}")
148
- try:
149
- # Fallback to stable model
150
- self.gliner_model = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
151
- print("✓ GLiNER fallback model loaded successfully")
152
- except Exception as e2:
153
- print(f"Error loading GLiNER model: {str(e2)}")
154
- return None
155
- return self.gliner_model
156
-
157
- def assign_colours(self, standard_entities, custom_entities):
158
- """Assign colours to all entity types"""
159
- self.all_entity_colors = {}
160
-
161
- # Assign common colours
162
- for entity in standard_entities:
163
- self.all_entity_colors[entity.upper()] = STANDARD_COLORS.get(entity.upper(), '#CCCCCC')
164
-
165
- # Assign custom colours
166
- for i, entity in enumerate(custom_entities):
167
- if i < len(CUSTOM_COLOR_PALETTE):
168
- self.all_entity_colors[entity.upper()] = CUSTOM_COLOR_PALETTE[i]
169
- else:
170
- # Generate random colour if we run out
171
- self.all_entity_colors[entity.upper()] = f"#{random.randint(0, 0xFFFFFF):06x}"
172
-
173
- return self.all_entity_colors
174
 
175
- def extract_entities_by_model(self, text, entity_types, model_name, threshold=0.3):
176
- """Extract entities using the specified model"""
177
- # Convert full names to abbreviations for model processing
178
- abbrev_types = []
179
- for entity in entity_types:
180
- if entity in self.entity_mapping:
181
- abbrev_types.append(self.entity_mapping[entity])
182
- else:
183
- abbrev_types.append(entity)
 
 
 
184
 
185
- if 'spacy' in model_name:
186
- return self.extract_spacy_entities(text, abbrev_types)
187
- elif 'flair' in model_name:
188
- return self.extract_flair_entities(text, abbrev_types, model_name)
189
- elif 'gliner' in model_name:
190
- return self.extract_gliner_entities(text, abbrev_types, threshold, is_custom=False)
191
- else:
192
- return []
193
-
194
- def extract_spacy_entities(self, text, entity_types):
195
- """Extract entities using spaCy"""
196
- model = self.load_spacy_model()
197
- if model is None:
198
- return []
199
-
200
- try:
201
- doc = model(text)
202
- entities = []
203
- for ent in doc.ents:
204
- if ent.label_ in entity_types:
205
- # Convert abbreviation back to full name for display
206
- display_label = self.abbrev_to_full.get(ent.label_, ent.label_)
207
- entities.append({
208
- 'text': ent.text,
209
- 'label': display_label,
210
- 'start': ent.start_char,
211
- 'end': ent.end_char,
212
- 'confidence': 1.0, # spaCy doesn't provide confidence scores
213
- 'source': 'spaCy'
214
- })
215
- return entities
216
- except Exception as e:
217
- print(f"Error with spaCy extraction: {str(e)}")
218
- return []
219
-
220
- def extract_flair_entities(self, text, entity_types, model_name):
221
- """Extract entities using Flair"""
222
- model = self.load_flair_model(model_name)
223
- if model is None:
224
- return []
225
-
226
- try:
227
- from flair.data import Sentence
228
- sentence = Sentence(text)
229
- model.predict(sentence)
230
- entities = []
231
- for entity in sentence.get_spans('ner'):
232
- # Map Flair labels to our common set
233
- label = entity.tag
234
- if label == 'PERSON':
235
- label = 'PER'
236
- elif label == 'ORGANIZATION':
237
- label = 'ORG'
238
- elif label == 'LOCATION':
239
- label = 'LOC'
240
- elif label == 'MISCELLANEOUS':
241
- label = 'MISC'
242
-
243
- if label in entity_types:
244
- # Convert abbreviation back to full name for display
245
- display_label = self.abbrev_to_full.get(label, label)
246
- entities.append({
247
- 'text': entity.text,
248
- 'label': display_label,
249
- 'start': entity.start_position,
250
- 'end': entity.end_position,
251
- 'confidence': entity.score,
252
- 'source': f'Flair-{model_name.split("-")[-1]}'
253
- })
254
- return entities
255
- except Exception as e:
256
- print(f"Error with Flair extraction: {str(e)}")
257
- return []
258
-
259
- def extract_gliner_entities(self, text, entity_types, threshold=0.3, is_custom=True):
260
- """Extract entities using GLiNER"""
261
- model = self.load_gliner_model()
262
- if model is None:
263
- return []
264
-
265
  try:
266
- entities = model.predict_entities(text, entity_types, threshold=threshold)
267
- result = []
268
- for entity in entities:
269
- # Convert abbreviation back to full name for display if not custom
270
- if not is_custom:
271
- display_label = self.abbrev_to_full.get(entity['label'].upper(), entity['label'].upper())
272
- else:
273
- display_label = entity['label'].upper()
274
-
275
- result.append({
276
- 'text': entity['text'],
277
- 'label': display_label,
278
- 'start': entity['start'],
279
- 'end': entity['end'],
280
- 'confidence': entity.get('score', 0.0),
281
- 'source': 'GLiNER-Custom' if is_custom else 'GLiNER-Common'
282
- })
283
- return result
284
- except Exception as e:
285
- print(f"Error with GLiNER extraction: {str(e)}")
286
- return []
287
-
288
- def find_overlapping_entities(entities):
289
- """Find and share overlapping entities - specifically entities found by BOTH common NER models AND custom entities"""
290
- if not entities:
291
- return []
292
-
293
- # Sort entities by start position
294
- sorted_entities = sorted(entities, key=lambda x: x['start'])
295
- shared_entities = []
296
-
297
- i = 0
298
- while i < len(sorted_entities):
299
- current_entity = sorted_entities[i]
300
- overlapping_entities = [current_entity]
301
-
302
- # Find all entities that overlap with current entity
303
- j = i + 1
304
- while j < len(sorted_entities):
305
- next_entity = sorted_entities[j]
306
-
307
- # Check if entities overlap (same text span or overlapping positions)
308
- if (current_entity['start'] <= next_entity['start'] < current_entity['end'] or
309
- next_entity['start'] <= current_entity['start'] < current_entity['end'] or
310
- current_entity['text'].lower() == next_entity['text'].lower()):
311
- overlapping_entities.append(next_entity)
312
- sorted_entities.pop(j)
313
- else:
314
- j += 1
315
 
316
- # Create shared entity only if we have BOTH common and custom entities
317
- if len(overlapping_entities) == 1:
318
- shared_entities.append(overlapping_entities[0])
 
 
 
 
 
 
 
 
 
 
319
  else:
320
- # Check if this is a true "shared" entity (common + custom)
321
- has_common = False
322
- has_custom = False
323
-
324
- for entity in overlapping_entities:
325
- source = entity.get('source', '')
326
- if source in ['spaCy', 'GLiNER-Common'] or source.startswith('Flair-'):
327
- has_common = True
328
- elif source == 'GLiNER-Custom':
329
- has_custom = True
330
-
331
- if has_common and has_custom:
332
- # This is a true shared entity (common + custom)
333
- shared_entity = share_entities(overlapping_entities)
334
- shared_entities.append(shared_entity)
335
  else:
336
- # These are just overlapping entities from the same source type, keep separate
337
- shared_entities.extend(overlapping_entities)
338
-
339
- i += 1
340
-
341
- return shared_entities
342
-
343
- def share_entities(entity_list):
344
- """Share multiple overlapping entities into one"""
345
- if len(entity_list) == 1:
346
- return entity_list[0]
347
-
348
- # Use the entity with the longest text span as the base
349
- base_entity = max(entity_list, key=lambda x: len(x['text']))
350
-
351
- # Collect all labels and sources
352
- labels = [entity['label'] for entity in entity_list]
353
- sources = [entity['source'] for entity in entity_list]
354
- confidences = [entity['confidence'] for entity in entity_list]
355
-
356
- return {
357
- 'text': base_entity['text'],
358
- 'start': base_entity['start'],
359
- 'end': base_entity['end'],
360
- 'labels': labels,
361
- 'sources': sources,
362
- 'confidences': confidences,
363
- 'is_shared': True,
364
- 'entity_count': len(entity_list)
365
- }
366
-
367
- def create_highlighted_html(text, entities, entity_colors):
368
- """Create HTML with highlighted entities"""
369
- if not entities:
370
- return f"<div style='padding: 15px; border: 1px solid #ddd; border-radius: 5px; background-color: #fafafa;'><p>{text}</p></div>"
371
-
372
- # Find and share overlapping entities
373
- shared_entities = find_overlapping_entities(entities)
374
-
375
- # Sort by start position
376
- sorted_entities = sorted(shared_entities, key=lambda x: x['start'])
377
-
378
- # Create HTML with highlighting
379
- html_parts = []
380
- last_end = 0
381
-
382
- for entity in sorted_entities:
383
- # Add text before entity
384
- html_parts.append(text[last_end:entity['start']])
385
-
386
- if entity.get('is_shared', False):
387
- # Handle shared entity with multiple colours
388
- html_parts.append(create_shared_entity_html(entity, entity_colors))
389
- else:
390
- # Handle single entity
391
- html_parts.append(create_single_entity_html(entity, entity_colors))
392
-
393
- last_end = entity['end']
394
-
395
- # Add remaining text
396
- html_parts.append(text[last_end:])
397
-
398
- highlighted_text = ''.join(html_parts)
399
-
400
- return f"""
401
- <div style='padding: 15px; border: 2px solid #ddd; border-radius: 8px; background-color: #fafafa; margin: 10px 0;'>
402
- <h4 style='margin: 0 0 15px 0; color: #333;'>📝 Text with Highlighted Entities</h4>
403
- <div style='line-height: 1.8; font-size: 16px; background-color: white; padding: 15px; border-radius: 5px;'>{highlighted_text}</div>
404
- </div>
405
- """
406
-
407
- def create_single_entity_html(entity, entity_colors):
408
- """Create HTML for a single entity"""
409
- label = entity['label']
410
- colour = entity_colors.get(label.upper(), '#CCCCCC')
411
- confidence = entity.get('confidence', 0.0)
412
- source = entity.get('source', 'Unknown')
413
-
414
- return (f'<span style="background-color: {colour}; padding: 2px 4px; '
415
- f'border-radius: 3px; margin: 0 1px; '
416
- f'border: 1px solid {colour}; color: white; font-weight: bold;" '
417
- f'title="{label} ({source}) - confidence: {confidence:.2f}">'
418
- f'{entity["text"]}</span>')
419
-
420
- def create_shared_entity_html(entity, entity_colors):
421
- """Create HTML for a shared entity with multiple colours"""
422
- labels = entity['labels']
423
- sources = entity['sources']
424
- confidences = entity['confidences']
425
-
426
- # Get colours for each label
427
- colours = []
428
- for label in labels:
429
- colour = entity_colors.get(label.upper(), '#CCCCCC')
430
- colours.append(colour)
431
-
432
- # Create gradient background
433
- if len(colours) == 2:
434
- gradient = f"linear-gradient(to right, {colours[0]} 50%, {colours[1]} 50%)"
435
- else:
436
- # For more colours, create equal segments
437
- segment_size = 100 / len(colours)
438
- gradient_parts = []
439
- for i, colour in enumerate(colours):
440
- start = i * segment_size
441
- end = (i + 1) * segment_size
442
- gradient_parts.append(f"{colour} {start}%, {colour} {end}%")
443
- gradient = f"linear-gradient(to right, {', '.join(gradient_parts)})"
444
-
445
- # Create tooltip
446
- tooltip_parts = []
447
- for i, label in enumerate(labels):
448
- tooltip_parts.append(f"{label} ({sources[i]}) - {confidences[i]:.2f}")
449
- tooltip = " | ".join(tooltip_parts)
450
-
451
- return (f'<span style="background: {gradient}; padding: 2px 4px; '
452
- f'border-radius: 3px; margin: 0 1px; '
453
- f'border: 2px solid #333; color: white; font-weight: bold;" '
454
- f'title="SHARED: {tooltip}">'
455
- f'{entity["text"]} 🤝</span>')
456
-
457
- def create_entity_table_html(entities_of_type, entity_type, colour, is_shared=False):
458
- """Create HTML table for a specific entity type"""
459
- if is_shared:
460
- table_html = f"""
461
- <table style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
462
- <thead>
463
- <tr style="background-color: {colour}; color: white;">
464
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Entity Text</th>
465
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">All Labels</th>
466
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Sources</th>
467
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Count</th>
468
- </tr>
469
- </thead>
470
- <tbody>
471
- """
472
-
473
- for entity in entities_of_type:
474
- labels_text = " | ".join(entity['labels'])
475
- sources_text = " | ".join(entity['sources'])
476
-
477
- table_html += f"""
478
- <tr style="background-color: #fff;">
479
- <td style="padding: 10px; border: 1px solid #ddd; font-weight: bold;">{entity['text']}</td>
480
- <td style="padding: 10px; border: 1px solid #ddd;">{labels_text}</td>
481
- <td style="padding: 10px; border: 1px solid #ddd;">{sources_text}</td>
482
- <td style="padding: 10px; border: 1px solid #ddd; text-align: center;">
483
- <span style='background-color: #28a745; color: white; padding: 2px 6px; border-radius: 10px; font-size: 11px;'>
484
- {entity['entity_count']}
485
- </span>
486
- </td>
487
- </tr>
488
- """
489
- else:
490
- table_html = f"""
491
- <table style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
492
- <thead>
493
- <tr style="background-color: {colour}; color: white;">
494
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Entity Text</th>
495
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Confidence</th>
496
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Type</th>
497
- <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Source</th>
498
- </tr>
499
- </thead>
500
- <tbody>
501
- """
502
-
503
- # Sort by confidence score
504
- entities_of_type.sort(key=lambda x: x.get('confidence', 0), reverse=True)
505
-
506
- for entity in entities_of_type:
507
- confidence = entity.get('confidence', 0.0)
508
- confidence_colour = "#28a745" if confidence > 0.7 else "#ffc107" if confidence > 0.4 else "#dc3545"
509
- source = entity.get('source', 'Unknown')
510
- source_badge = f"<span style='background-color: #007bff; color: white; padding: 2px 6px; border-radius: 10px; font-size: 11px;'>{source}</span>"
511
-
512
- table_html += f"""
513
- <tr style="background-color: #fff;">
514
- <td style="padding: 10px; border: 1px solid #ddd; font-weight: bold;">{entity['text']}</td>
515
- <td style="padding: 10px; border: 1px solid #ddd;">
516
- <span style="color: {confidence_colour}; font-weight: bold;">
517
- {confidence:.3f}
518
- </span>
519
- </td>
520
- <td style="padding: 10px; border: 1px solid #ddd;">{entity['label']}</td>
521
- <td style="padding: 10px; border: 1px solid #ddd;">{source_badge}</td>
522
- </tr>
523
- """
524
 
525
- table_html += "</tbody></table>"
526
- return table_html
527
-
528
- def create_all_entity_tables(entities, entity_colors):
529
- """Create all entity tables in a single container"""
530
- if not entities:
531
- return "<p style='text-align: center; padding: 20px;'>No entities found.</p>"
532
 
533
- # Share overlapping entities
534
- shared_entities = find_overlapping_entities(entities)
535
-
536
- # Group entities by type
537
- entity_groups = {}
538
- for entity in shared_entities:
539
- if entity.get('is_shared', False):
540
- key = 'SHARED_ENTITIES'
541
- else:
542
- key = entity['label']
543
-
544
- if key not in entity_groups:
545
- entity_groups[key] = []
546
- entity_groups[key].append(entity)
547
-
548
- if not entity_groups:
549
- return "<p style='text-align: center; padding: 20px;'>No entities found.</p>"
550
-
551
- # Create container with all tables
552
- all_tables_html = """
553
- <div style='max-height: 600px; overflow-y: auto; border: 2px solid #ddd; border-radius: 8px; padding: 20px; background-color: #fafafa;'>
554
- <style>
555
- .entity-section {
556
- margin-bottom: 30px;
557
- background-color: white;
558
- border-radius: 8px;
559
- padding: 15px;
560
- box-shadow: 0 2px 4px rgba(0,0,0,0.1);
561
- }
562
- .entity-section:last-child {
563
- margin-bottom: 0;
564
- }
565
- .section-header {
566
- display: flex;
567
- align-items: center;
568
- margin-bottom: 15px;
569
- padding-bottom: 10px;
570
- border-bottom: 2px solid #eee;
571
- }
572
- .entity-count {
573
- background-color: #007bff;
574
- color: white;
575
- padding: 4px 12px;
576
- border-radius: 15px;
577
- font-size: 14px;
578
- font-weight: bold;
579
- margin-left: 10px;
580
- }
581
- .quick-nav {
582
- position: sticky;
583
- top: 0;
584
- background-color: #f8f9fa;
585
- padding: 10px;
586
- margin-bottom: 20px;
587
- border-radius: 8px;
588
- display: flex;
589
- flex-wrap: wrap;
590
- gap: 8px;
591
- z-index: 10;
592
- box-shadow: 0 2px 4px rgba(0,0,0,0.1);
593
- }
594
- .nav-button {
595
- padding: 6px 12px;
596
- border: 1px solid #ddd;
597
- border-radius: 20px;
598
- background-color: white;
599
- cursor: pointer;
600
- transition: all 0.3s;
601
- text-decoration: none;
602
- color: #333;
603
- font-size: 13px;
604
- font-weight: 500;
605
- }
606
- .nav-button:hover {
607
- background-color: #4ECDC4;
608
- color: white;
609
- border-color: #4ECDC4;
610
- }
611
- </style>
612
- """
613
 
614
- # Create quick navigation
615
- all_tables_html += '<div class="quick-nav">'
616
- all_tables_html += '<strong style="margin-right: 10px;">Quick Navigation:</strong>'
617
 
618
- # Sort entity groups to show shared entities first
619
- sorted_groups = []
620
- if 'SHARED_ENTITIES' in entity_groups:
621
- sorted_groups.append(('SHARED_ENTITIES', entity_groups['SHARED_ENTITIES']))
 
 
622
 
623
- for entity_type, entities_list in sorted(entity_groups.items()):
624
- if entity_type != 'SHARED_ENTITIES':
625
- sorted_groups.append((entity_type, entities_list))
 
 
 
626
 
627
- for entity_type, entities_list in sorted_groups:
628
- if entity_type == 'SHARED_ENTITIES':
629
- icon = '🤝'
630
- label = 'Shared'
631
- else:
632
- icon = '🎯' if entity_type in STANDARD_ENTITIES else '✨'
633
- label = entity_type
634
-
635
- all_tables_html += f'<a href="#{entity_type.replace(" ", "_")}" class="nav-button">{icon} {label} ({len(entities_list)})</a>'
636
 
637
- all_tables_html += '</div>'
 
638
 
639
- # Add shared entities section if any
640
- if 'SHARED_ENTITIES' in entity_groups:
641
- shared_entities_list = entity_groups['SHARED_ENTITIES']
642
- all_tables_html += f"""
643
- <div class="entity-section" id="SHARED_ENTITIES">
644
- <div class="section-header">
645
- <h3 style="margin: 0; display: flex; align-items: center;">
646
- <span style="font-size: 24px; margin-right: 10px;">🤝</span>
647
- Shared Entities
648
- <span class="entity-count">{len(shared_entities_list)} found</span>
649
- </h3>
650
- </div>
651
- {create_entity_table_html(shared_entities_list, 'SHARED_ENTITIES', '#666666', is_shared=True)}
652
- </div>
653
- """
654
 
655
- # Add other entity types
656
- for entity_type, entities_of_type in sorted(entity_groups.items()):
657
- if entity_type == 'SHARED_ENTITIES':
658
- continue
659
-
660
- colour = entity_colors.get(entity_type.upper(), '#f0f0f0')
661
- is_standard = entity_type in STANDARD_ENTITIES
662
- icon = "🎯" if is_standard else "✨"
663
- type_label = "Common NER" if is_standard else "Custom GLiNER"
664
-
665
- all_tables_html += f"""
666
- <div class="entity-section" id="{entity_type.replace(' ', '_')}">
667
- <div class="section-header">
668
- <h3 style="margin: 0; display: flex; align-items: center;">
669
- <span style="font-size: 24px; margin-right: 10px;">{icon}</span>
670
- {entity_type}
671
- <span class="entity-count" style="background-color: {colour};">{len(entities_of_type)} found</span>
672
- </h3>
673
- <span style="margin-left: auto; color: #666; font-size: 14px;">{type_label}</span>
674
- </div>
675
- {create_entity_table_html(entities_of_type, entity_type, colour)}
676
- </div>
677
- """
678
 
679
- all_tables_html += "</div>"
 
680
 
681
- return all_tables_html
682
-
683
- def create_legend_html(entity_colors, standard_entities, custom_entities):
684
- """Create a legend showing entity colours"""
685
- if not entity_colors:
686
- return ""
687
-
688
- html = "<div style='margin: 15px 0; padding: 15px; background-color: #f8f9fa; border-radius: 8px;'>"
689
- html += "<h4 style='margin: 0 0 15px 0;'>🎨 Entity Type Legend</h4>"
690
-
691
- if standard_entities:
692
- html += "<div style='margin-bottom: 15px;'>"
693
- html += "<h5 style='margin: 0 0 8px 0;'>🎯 Common Entities:</h5>"
694
- html += "<div style='display: flex; flex-wrap: wrap; gap: 8px;'>"
695
- for entity_type in standard_entities:
696
- colour = entity_colors.get(entity_type.upper(), '#ccc')
697
- html += f"<span style='background-color: {colour}; padding: 4px 8px; border-radius: 15px; color: white; font-weight: bold; font-size: 12px;'>{entity_type}</span>"
698
- html += "</div></div>"
699
-
700
- if custom_entities:
701
- html += "<div>"
702
- html += "<h5 style='margin: 0 0 8px 0;'>✨ Custom Entities:</h5>"
703
- html += "<div style='display: flex; flex-wrap: wrap; gap: 8px;'>"
704
- for entity_type in custom_entities:
705
- colour = entity_colors.get(entity_type.upper(), '#ccc')
706
- html += f"<span style='background-color: {colour}; padding: 4px 8px; border-radius: 15px; color: white; font-weight: bold; font-size: 12px;'>{entity_type}</span>"
707
- html += "</div></div>"
708
-
709
- html += "</div>"
710
- return html
711
-
712
- # Initialize the NER manager
713
- ner_manager = HybridNERManager()
714
-
715
- def process_text(text, standard_entities, custom_entities_str, confidence_threshold, selected_model, progress=gr.Progress()):
716
- """Main processing function for Gradio interface with progress tracking"""
717
- if not text.strip():
718
- return "❌ Please enter some text to analyse", "", "", gr.update(visible=False)
719
-
720
- progress(0.1, desc="Initialising...")
721
-
722
- # Parse custom entities
723
- custom_entities = []
724
- if custom_entities_str.strip():
725
- custom_entities = [entity.strip() for entity in custom_entities_str.split(',') if entity.strip()]
726
-
727
- # Parse common entities
728
- selected_standard = [entity for entity in standard_entities if entity]
729
-
730
- if not selected_standard and not custom_entities:
731
- return "❌ Please select at least one common entity type OR enter custom entity types", "", "", gr.update(visible=False)
732
-
733
- progress(0.2, desc="Loading models...")
734
-
735
- all_entities = []
736
-
737
- # Extract common entities using selected model
738
- if selected_standard and selected_model:
739
- progress(0.4, desc="Extracting common entities...")
740
- standard_entities_results = ner_manager.extract_entities_by_model(text, selected_standard, selected_model, confidence_threshold)
741
- all_entities.extend(standard_entities_results)
742
-
743
- # Extract custom entities using GLiNER
744
- if custom_entities:
745
- progress(0.6, desc="Extracting custom entities...")
746
- custom_entity_results = ner_manager.extract_gliner_entities(text, custom_entities, confidence_threshold, is_custom=True)
747
- all_entities.extend(custom_entity_results)
748
-
749
- if not all_entities:
750
- return "❌ No entities found. Try lowering the confidence threshold or using different entity types.", "", "", gr.update(visible=False)
751
-
752
- progress(0.8, desc="Processing results...")
753
 
754
- # Assign colours
755
- entity_colors = ner_manager.assign_colours(selected_standard, custom_entities)
 
756
 
757
- # Create outputs
758
- legend_html = create_legend_html(entity_colors, selected_standard, custom_entities)
759
- highlighted_html = create_highlighted_html(text, all_entities, entity_colors)
760
- results_html = create_all_entity_tables(all_entities, entity_colors)
761
 
762
- progress(0.9, desc="Creating summary...")
763
-
764
- # Create summary with shared entities terminology
765
- total_entities = len(all_entities)
766
- shared_entities = find_overlapping_entities(all_entities)
767
- final_count = len(shared_entities)
768
- shared_count = sum(1 for e in shared_entities if e.get('is_shared', False))
769
 
770
- summary = f"""
771
- ## 📊 Analysis Summary
772
- - **Total entities found:** {total_entities}
773
- - **Final entities displayed:** {final_count}
774
- - **Shared entities:** {shared_count}
775
- - **Average confidence:** {sum(e.get('confidence', 0) for e in all_entities) / total_entities:.3f}
776
- """
777
-
778
- progress(1.0, desc="Complete!")
779
 
780
- return summary, legend_html + highlighted_html, results_html, gr.update(visible=True)
781
 
782
- # Create Gradio interface
783
  def create_interface():
784
- with gr.Blocks(title="Hybrid NER + GLiNER Tool", theme=gr.themes.Soft()) as demo:
785
  gr.Markdown("""
786
- # Named Entity Recognition (NER) Explorer Tool
787
 
788
- Combine common Named Entity Recognition (NER) categories with your own custom entity types!
789
- This tool uses both traditional NER models and GLiNER for comprehensive entity extraction, allowing you to explore what NER looks like in practice.
790
-
791
- ### How to use this tool:
792
- 1. **📝 Enter your text** in the text area below
793
- 2. **🎯 Select a model** from the dropdown menu
794
- 3. **☑️ Select common entity types** you want to identify (PERSON, ORGANIZATION, LOCATION, etc.)
795
- 4. **✨ Add custom entities** (comma-separated) like "relationships, occupations, skills"
796
- 5. **⚙️ Adjust confidence threshold**
797
- 6. **🔍 Click "Analyse Text"** to see results
798
- (NB: common/custom entities which overlap are shown with split-colour highlighting)
799
- 7. **🔄 Refresh the page** to try again with new text
800
- """)
801
- # Add tip box
802
- gr.HTML("""
803
- <div style="background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 8px; padding: 12px; margin: 15px 0;">
804
- <strong style="color: #856404;">💡 Top tip:</strong> No model is perfect - all can miss and/or incorrectly identify entity types.
805
- </div>
806
  """)
807
 
808
  with gr.Row():
809
- with gr.Column(scale=2):
810
  text_input = gr.Textbox(
811
- label="📝 Text to Analyse",
812
- placeholder="Enter your text here...",
813
- lines=12,
814
  max_lines=20
815
  )
816
 
817
  with gr.Column(scale=1):
818
- confidence_threshold = gr.Slider(
819
- minimum=0.1,
820
- maximum=0.9,
821
- value=0.3,
822
- step=0.1,
823
- label="🎚️ Confidence Threshold"
 
 
 
 
 
824
  )
825
-
826
- # Add confidence threshold explanation
827
- gr.HTML("""
828
- <details style="margin: 10px 0; padding: 10px; background-color: #f8f9fa; border-radius: 8px; border: 1px solid #ddd;">
829
- <summary style="cursor: pointer; font-weight: bold; padding: 5px; color: #1976d2;">
830
- ℹ️ Understanding the Confidence Threshold
831
- </summary>
832
- <div style="margin-top: 10px; padding: 10px; font-size: 14px;">
833
- <p style="margin: 0 0 10px 0;">The <strong>confidence threshold</strong> controls how certain the model needs to be before identifying an entity:</p>
834
- <ul style="margin: 0; padding-left: 20px;">
835
- <li><strong>Lower values (0.1-0.3):</strong> More entities detected, but may include false positives</li>
836
- <li><strong>Medium values (0.4-0.6):</strong> Balanced detection with moderate confidence</li>
837
- <li><strong>Higher values (0.7-0.9):</strong> Only highly confident entities detected, may miss some valid entities</li>
838
- </ul>
839
- <p style="margin: 10px 0 0 0; font-style: italic;"><strong>Top Tip:</strong> Start with 0.3 for comprehensive detection, then adjust based on your needs.</p>
840
- </div>
841
- </details>
842
- """)
843
 
844
  with gr.Row():
845
- with gr.Column():
846
- gr.Markdown("### 🎯 Common Entity Types")
847
-
848
- # Model selector
849
- model_dropdown = gr.Dropdown(
850
- choices=ner_manager.model_names,
851
- value=ner_manager.model_names[0],
852
- label="Select NER Model for Common Entities"
853
- )
854
-
855
- # Common entities with select all functionality
856
- standard_entities = gr.CheckboxGroup(
857
- choices=STANDARD_ENTITIES,
858
- value=['PERSON', 'ORGANIZATION', 'LOCATION', 'MISCELLANEOUS'], # Default selection with full names
859
- label="Select Common Entities"
860
- )
861
-
862
- # Select/Deselect All button
863
- with gr.Row():
864
- select_all_btn = gr.Button("🔘 Deselect All", size="sm")
865
-
866
- # Function for select/deselect all
867
- def toggle_all_entities(current_selection):
868
- if len(current_selection) > 0:
869
- # If any are selected, deselect all
870
- return [], "☑️ Select All"
871
- else:
872
- # If none selected, select all
873
- return STANDARD_ENTITIES, "🔘 Deselect All"
874
-
875
- select_all_btn.click(
876
- fn=toggle_all_entities,
877
- inputs=[standard_entities],
878
- outputs=[standard_entities, select_all_btn]
879
- )
880
-
881
- with gr.Column():
882
- gr.Markdown("### ✨ Custom Entity Types - Powered by GLiNER")
883
- custom_entities = gr.Textbox(
884
- label="Custom Entities (comma-separated)",
885
- placeholder="e.g. relationships, civilian occupations, emotions",
886
- lines=3
887
- )
888
- gr.Markdown("""
889
- **Examples:**
890
- - relationships, occupations, skills
891
- - emotions, actions, objects
892
- - medical conditions, treatments
893
-
894
-
895
-
896
- """)
897
-
898
- # Add glossary (alphabetically ordered)
899
- gr.HTML("""
900
- <details style="margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 8px; border: 1px solid #ddd;">
901
- <summary style="cursor: pointer; font-weight: bold; padding: 5px; color: #1976d2;">
902
- ℹ️ Entity Type Definitions
903
- </summary>
904
- <div style="margin-top: 10px; padding: 10px;">
905
- <dl style="margin: 0; font-size: 14px;">
906
- <div style="margin-bottom: 8px;">
907
- <dt style="font-weight: bold; display: inline; color: #FF6B6B;">Date (DATE):</dt>
908
- <dd style="display: inline; margin-left: 5px;">Absolute or relative dates or periods</dd>
909
- </div>
910
- <div style="margin-bottom: 8px;">
911
- <dt style="font-weight: bold; display: inline; color: #4ECDC4;">Event (EVENT):</dt>
912
- <dd style="display: inline; margin-left: 5px;">Named hurricanes, battles, wars, sports events, etc.</dd>
913
- </div>
914
- <div style="margin-bottom: 8px;">
915
- <dt style="font-weight: bold; display: inline; color: #45B7D1;">Facility (FAC):</dt>
916
- <dd style="display: inline; margin-left: 5px;">Buildings, airports, highways, bridges, etc.</dd>
917
- </div>
918
- <div style="margin-bottom: 8px;">
919
- <dt style="font-weight: bold; display: inline; color: #F9CA24;">Geopolitical Entity (GPE):</dt>
920
- <dd style="display: inline; margin-left: 5px;">Countries, cities, states</dd>
921
- </div>
922
- <div style="margin-bottom: 8px;">
923
- <dt style="font-weight: bold; display: inline; color: #6C5CE7;">Language (LANG):</dt>
924
- <dd style="display: inline; margin-left: 5px;">Any named language</dd>
925
- </div>
926
- <div style="margin-bottom: 8px;">
927
- <dt style="font-weight: bold; display: inline; color: #A0E7E5;">Location (LOC):</dt>
928
- <dd style="display: inline; margin-left: 5px;">Non-GPE locations - Mountain ranges, bodies of water</dd>
929
- </div>
930
- <div style="margin-bottom: 8px;">
931
- <dt style="font-weight: bold; display: inline; color: #FD79A8;">Miscellaneous (MISC):</dt>
932
- <dd style="display: inline; margin-left: 5px;">Entities that don't fit elsewhere</dd>
933
- </div>
934
- <div style="margin-bottom: 8px;">
935
- <dt style="font-weight: bold; display: inline; color: #8E8E93;">Nationalities/Groups (NORP):</dt>
936
- <dd style="display: inline; margin-left: 5px;">Nationalities or religious or political groups</dd>
937
- </div>
938
- <div style="margin-bottom: 8px;">
939
- <dt style="font-weight: bold; display: inline; color: #55A3FF;">Organization (ORG):</dt>
940
- <dd style="display: inline; margin-left: 5px;">Companies, agencies, institutions, etc.</dd>
941
- </div>
942
- <div style="margin-bottom: 8px;">
943
- <dt style="font-weight: bold; display: inline; color: #00B894;">Person (PER):</dt>
944
- <dd style="display: inline; margin-left: 5px;">People, including fictional characters</dd>
945
- </div>
946
- <div style="margin-bottom: 8px;">
947
- <dt style="font-weight: bold; display: inline; color: #E17055;">Product (PRODUCT):</dt>
948
- <dd style="display: inline; margin-left: 5px;">Objects, vehicles, foods, etc. (Not services)</dd>
949
- </div>
950
- <div style="margin-bottom: 8px;">
951
- <dt style="font-weight: bold; display: inline; color: #DDA0DD;">Work of Art (Work of Art):</dt>
952
- <dd style="display: inline; margin-left: 5px;">Titles of books, songs, movies, paintings, etc.</dd>
953
- </div>
954
- </dl>
955
- </div>
956
- </details>
957
- """)
958
-
959
- analyse_btn = gr.Button("🔍 Analyse Text", variant="primary", size="lg")
960
 
961
- # Add horizontal line separator after the button
962
- gr.HTML("<hr style='margin-top: 20px; margin-bottom: 15px;'>")
963
-
964
- # Output sections
965
  with gr.Row():
966
- summary_output = gr.Markdown(label="Summary")
967
 
968
  with gr.Row():
969
- highlighted_output = gr.HTML(label="Highlighted Text")
970
-
971
- # Results section (initially hidden)
972
- with gr.Row(visible=False) as results_row:
973
- with gr.Column():
974
- gr.Markdown("### 📋 Detailed Results")
975
- results_output = gr.HTML(label="Entity Results")
976
-
977
- # Connect the button to the processing function
978
- analyse_btn.click(
979
- fn=process_text,
980
- inputs=[
981
- text_input,
982
- standard_entities,
983
- custom_entities,
984
- confidence_threshold,
985
- model_dropdown
986
- ],
987
- outputs=[summary_output, highlighted_output, results_output, results_row]
988
- )
989
-
990
- # Updated examples text with reduced spacing
991
- with gr.Column():
992
- gr.Markdown("""
993
- ### 💡 No example text to test? No problem!
994
- Simply click on one of the examples provided below, and the fields will be populated for you.
995
- """, elem_id="examples-heading")
996
 
 
 
 
 
 
 
 
 
 
997
  gr.Examples(
998
  examples=[
999
  [
1000
- "On June 6, 1944, Allied forces launched Operation Overlord, the invasion of Normandy. General Dwight D. Eisenhower commanded the operation, while Field Marshal Bernard Montgomery led ground forces. The BBC broadcast coded messages to the French Resistance, including the famous line 'The long sobs of autumn violins.'",
1001
- ["PERSON", "ORGANIZATION", "LOCATION", "DATE", "EVENT"],
1002
- "military operations, military ranks, historical battles",
1003
- 0.3,
1004
- "spacy_en_core_web_trf"
1005
  ],
1006
  [
1007
- "In Jane Austen's 'Pride and Prejudice', Elizabeth Bennet first meets Mr. Darcy at the Meryton assembly. The novel, published in 1813, explores themes of marriage and social class in Regency England. Austen wrote to her sister Cassandra about the manuscript while staying at Chawton Cottage.",
1008
- ["PERSON", "LOCATION", "DATE", "WORK OF ART"],
1009
- "literary themes, relationships, literary periods",
1010
- 0.3,
1011
- "gliner_knowledgator/modern-gliner-bi-large-v1.0"
1012
  ],
1013
  [
1014
- "Charles Darwin arrived at the Galápagos Islands aboard HMS Beagle in September 1835. During his five-week visit, Darwin collected specimens of finches, tortoises, and mockingbirds. His observations of these species' variations across different islands later contributed to his theory of evolution by natural selection, published in 'On the Origin of Species' in 1859.",
1015
- ["PERSON", "LOCATION", "DATE", "WORK OF ART", "PRODUCT"],
1016
- "scientific concepts, species, research methods",
1017
- 0.3,
1018
- "flair_ner-large"
1019
  ]
1020
  ],
1021
- inputs=[
1022
- text_input,
1023
- standard_entities,
1024
- custom_entities,
1025
- confidence_threshold,
1026
- model_dropdown
1027
- ],
1028
- label="Examples"
1029
  )
1030
 
1031
- # Add custom CSS to make Examples label black and improve entity label readability
1032
- gr.HTML("""
1033
- <style>
1034
- /* Make the Examples label text black */
1035
- .gradio-examples-label {
1036
- color: black !important;
1037
- }
1038
- /* Also target the label more specifically if needed */
1039
- h4.examples-label, .examples-label {
1040
- color: black !important;
1041
- }
1042
- /* Override any link colors in the examples section */
1043
- #examples-heading + div label,
1044
- #examples-heading + div .label-text {
1045
- color: black !important;
1046
- }
1047
-
1048
- /* Improve readability of entity checkbox labels */
1049
- .gradio-checkbox-group label {
1050
- font-size: 14px !important;
1051
- font-weight: normal !important;
1052
- color: #333333 !important;
1053
- line-height: 1.5 !important;
1054
- }
1055
-
1056
- /* Make the checkbox labels match the entity definitions style */
1057
- input[type="checkbox"] + span {
1058
- font-size: 14px !important;
1059
- color: #333333 !important;
1060
- }
1061
- </style>
 
 
 
 
 
1062
  """)
1063
 
1064
- # Add model information links
1065
  gr.HTML("""
1066
- <hr style="margin-top: 40px; margin-bottom: 20px;">
1067
- <div style="background-color: #f8f9fa; padding: 20px; border-radius: 8px; margin-top: 20px;">
1068
- <h4 style="margin-top: 0;">📚 Model Information & Documentation (incl. details on usage terms)</h4>
1069
- <p style="font-size: 14px; margin-bottom: 15px;">Learn more about the models used in this tool:</p>
1070
- <ul style="font-size: 14px; line-height: 1.8;">
1071
- <li><strong>flair_ner-large:</strong>
1072
- <a href="https://huggingface.co/flair/ner-english-large" target="_blank" style="color: #1976d2;">
1073
- Flair NER English Large Model ↗
1074
- </a>
1075
- </li>
1076
- <li><strong>spacy_en_core_web_trf:</strong>
1077
- <a href="https://spacy.io/models/en#en_core_web_trf" target="_blank" style="color: #1976d2;">
1078
- spaCy English Transformer Model ↗
1079
- </a>
1080
- </li>
1081
- <li><strong>flair_ner-ontonotes-large:</strong>
1082
- <a href="https://huggingface.co/flair/ner-english-ontonotes-large" target="_blank" style="color: #1976d2;">
1083
- Flair OntoNotes Large Model ↗
1084
- </a>
1085
- </li>
1086
- <li><strong>gliner_knowledgator/modern-gliner-bi-large-v1.0:</strong>
1087
- <a href="https://github.com/urchade/GLiNER/blob/main/README_Extended.md" target="_blank" style="color: #1976d2;">
1088
- GLiNER Extended Documentation ↗
1089
- </a>
1090
- </li>
1091
- </ul>
1092
  </div>
1093
-
1094
- <br>
1095
- <hr style="margin-top: 40px; margin-bottom: 20px;">
1096
- <div style="background-color: #f8f9fa; padding: 20px; border-radius: 8px; margin-top: 20px; text-align: center;">
1097
- <p style="font-size: 14px; line-height: 1.8; margin: 0;">
1098
- This <strong>NER Explorer Tool</strong> was created as part of the
1099
- <a href="https://digitalscholarship.web.ox.ac.uk/" target="_blank" style="color: #1976d2;">
1100
- Digital Scholarship at Oxford (DiSc)
1101
- </a>
1102
- funded research project:<br>
1103
- <em>Extracting Keywords from Crowdsourced Collections</em>.
1104
- </p><br><br>
1105
- <p style="font-size: 14px; line-height: 1.8; margin: 0;">
1106
- The code for this tool was built with the aid of Claude Opus 4.
1107
- </p>
1108
- </div>
1109
- """)
1110
-
1111
  return demo
1112
 
1113
  if __name__ == "__main__":
 
1
  import gradio as gr
 
 
 
 
 
2
  import re
3
+ import pandas as pd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
 
5
+ def parse_keywords_dict(keywords_text):
6
+ """Parse the keyword dictionary from text input"""
7
+ keywords_dict = {}
8
+
9
+ if not keywords_text.strip():
10
+ return keywords_dict
11
+
12
+ lines = keywords_text.strip().split('\n')
13
+ for line in lines:
14
+ line = line.strip()
15
+ if not line or '|' not in line:
16
+ continue
17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  try:
19
+ primary, secondary = line.split('|', 1)
20
+ primary = primary.strip()
21
+ secondary_list = [keyword.strip() for keyword in secondary.split(';') if keyword.strip()]
22
+ keywords_dict[primary] = secondary_list
23
+ except:
24
+ continue
25
+
26
+ return keywords_dict
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
+ def find_keywords(story, keywords_dict):
29
+ """Find keywords in the story text"""
30
+ if not story or not isinstance(story, str):
31
+ return ''
32
+
33
+ found_keywords = set()
34
+
35
+ # Search for each primary keyword and its synonyms
36
+ for primary_keyword, synonyms in keywords_dict.items():
37
+ # Special handling for 'US' to avoid matching 'us'
38
+ if primary_keyword.upper() == "US":
39
+ if ' US ' in story or story.startswith('US ') or story.endswith(' US'):
40
+ found_keywords.update(synonyms)
41
  else:
42
+ pattern = r'\b' + re.escape(primary_keyword) + r'\b'
43
+ if re.search(pattern, story, re.IGNORECASE):
44
+ found_keywords.update(synonyms)
45
+
46
+ # Check each secondary keyword independently
47
+ for synonym in synonyms:
48
+ if synonym.upper() == "US":
49
+ if ' US ' in story or story.startswith('US ') or story.endswith(' US'):
50
+ found_keywords.add(synonym)
 
 
 
 
 
 
51
  else:
52
+ if re.search(r'\b' + re.escape(synonym) + r'\b', story, re.IGNORECASE):
53
+ found_keywords.add(synonym)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
+ return '; '.join(sorted(found_keywords))
 
 
 
 
 
 
56
 
57
+ def highlight_keywords_in_text(text, keywords_list):
58
+ """Create HTML with highlighted keywords"""
59
+ if not keywords_list:
60
+ return text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
 
62
+ highlighted_text = text
63
+ colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#F9CA24', '#6C5CE7', '#A0E7E5', '#FD79A8', '#55A3FF', '#00B894', '#E17055']
 
64
 
65
+ for i, keyword in enumerate(keywords_list):
66
+ if keyword:
67
+ color = colors[i % len(colors)]
68
+ pattern = r'\b' + re.escape(keyword) + r'\b'
69
+ replacement = f'<span style="background-color: {color}; padding: 2px 4px; border-radius: 3px; color: white; font-weight: bold;">{keyword}</span>'
70
+ highlighted_text = re.sub(pattern, replacement, highlighted_text, flags=re.IGNORECASE)
71
 
72
+ return highlighted_text
73
+
74
+ def process_text(input_text, keywords_text):
75
+ """Main processing function"""
76
+ if not input_text.strip():
77
+ return "Please enter some text to analyze", "", "No keywords found"
78
 
79
+ if not keywords_text.strip():
80
+ return "Please enter keyword mappings", "", "No keyword dictionary provided"
 
 
 
 
 
 
 
81
 
82
+ # Parse keywords dictionary
83
+ keywords_dict = parse_keywords_dict(keywords_text)
84
 
85
+ if not keywords_dict:
86
+ return "Invalid keyword format. Please check your keyword dictionary format.", "", "Error parsing keywords"
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
+ # Find keywords in the text
89
+ found_keywords_str = find_keywords(input_text, keywords_dict)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
 
91
+ if not found_keywords_str:
92
+ return f"No keywords found in the text.\n\nKeyword dictionary loaded: {len(keywords_dict)} primary keywords", input_text, "No matches found"
93
 
94
+ # Create highlighted version
95
+ keywords_list = found_keywords_str.split('; ')
96
+ highlighted_html = highlight_keywords_in_text(input_text, keywords_list)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
 
98
+ # Create results summary
99
+ results_summary = f"""
100
+ ## Results Summary
101
 
102
+ **Keywords Found:** {len(keywords_list)}
103
+ **Matched Keywords:** {found_keywords_str}
 
 
104
 
105
+ **Keyword Dictionary Stats:**
106
+ - Primary keywords loaded: {len(keywords_dict)}
107
+ - Total searchable terms: {sum(len(synonyms) + 1 for synonyms in keywords_dict.values())}
 
 
 
 
108
 
109
+ **Copy this result to your spreadsheet:**
110
+ {found_keywords_str}
111
+ """
 
 
 
 
 
 
112
 
113
+ return results_summary, highlighted_html, found_keywords_str
114
 
115
+ # Create the Gradio interface
116
  def create_interface():
117
+ with gr.Blocks(title="Keyword Tagging Tool", theme=gr.themes.Soft()) as demo:
118
  gr.Markdown("""
119
+ # Keyword Tagging Tool
120
 
121
+ This tool matches text against a controlled vocabulary and returns all associated keywords.
122
+ Based on the keyword matching logic used for digital humanities research.
123
+
124
+ ## How to use:
125
+ 1. **Enter your text** in the left panel
126
+ 2. **Define your keyword dictionary** in the right panel using the format: `Primary Keyword | synonym1; synonym2; synonym3`
127
+ 3. **Click "Find Keywords"** to see results
128
+ 4. **Copy the results** to paste into your spreadsheet
 
 
 
 
 
 
 
 
 
 
129
  """)
130
 
131
  with gr.Row():
132
+ with gr.Column(scale=1):
133
  text_input = gr.Textbox(
134
+ label="Text to Analyze",
135
+ placeholder="Enter the text you want to tag with keywords...",
136
+ lines=15,
137
  max_lines=20
138
  )
139
 
140
  with gr.Column(scale=1):
141
+ keywords_input = gr.Textbox(
142
+ label="Keyword Dictionary",
143
+ placeholder="""Enter one mapping per line:
144
+ Primary Keyword | synonym1; synonym2; synonym3
145
+
146
+ Example:
147
+ Prisoner of War | Prisoner of War; POW; POWs
148
+ United States | United States; USA; US; America
149
+ """,
150
+ lines=15,
151
+ max_lines=20
152
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
 
154
  with gr.Row():
155
+ find_btn = gr.Button("Find Keywords", variant="primary", size="lg")
156
+ clear_btn = gr.Button("Clear All", size="lg")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
 
 
 
 
 
158
  with gr.Row():
159
+ results_output = gr.Markdown(label="Results Summary")
160
 
161
  with gr.Row():
162
+ highlighted_output = gr.HTML(label="Text with Highlighted Keywords")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
 
164
+ with gr.Row():
165
+ copy_output = gr.Textbox(
166
+ label="Keywords for Spreadsheet (copy this text)",
167
+ lines=3,
168
+ max_lines=5
169
+ )
170
+
171
+ # Examples section
172
+ gr.Markdown("### Examples")
173
  gr.Examples(
174
  examples=[
175
  [
176
+ "During World War II, many prisoners of war were held in camps across Europe. The Geneva Convention established rules for POW treatment. American soldiers and British troops were among those captured.",
177
+ """Prisoner of War | Prisoner of War; POW; POWs; prisoner of war
178
+ World War II | World War II; WWII; Second World War
179
+ United States | United States; USA; US; America; American
180
+ United Kingdom | United Kingdom; UK; Britain; British"""
181
  ],
182
  [
183
+ "The University of Oxford is located in Oxford, England. Students from around the world study at this prestigious institution. The university has many colleges including Christ Church and Magdalen College.",
184
+ """University | University; university; institution; college
185
+ Oxford | Oxford; oxford
186
+ England | England; england; English
187
+ Student | Student; student; students; pupils"""
188
  ],
189
  [
190
+ "Shakespeare wrote many famous plays including Hamlet, Romeo and Juliet, and Macbeth. These works are performed in theatres worldwide and studied in schools.",
191
+ """William Shakespeare | Shakespeare; william shakespeare; playwright
192
+ Theatre | Theatre; theater; stage; performance
193
+ Play | Play; plays; drama; theatrical work
194
+ School | School; schools; education; academic"""
195
  ]
196
  ],
197
+ inputs=[text_input, keywords_input],
198
+ label="Click an example to try it out"
 
 
 
 
 
 
199
  )
200
 
201
+ # Button functions
202
+ find_btn.click(
203
+ fn=process_text,
204
+ inputs=[text_input, keywords_input],
205
+ outputs=[results_output, highlighted_output, copy_output]
206
+ )
207
+
208
+ def clear_all():
209
+ return "", "", "", "", ""
210
+
211
+ clear_btn.click(
212
+ fn=clear_all,
213
+ outputs=[text_input, keywords_input, results_output, highlighted_output, copy_output]
214
+ )
215
+
216
+ # Instructions
217
+ gr.Markdown("""
218
+ ## Format Guide
219
+
220
+ **Keyword Dictionary Format:**
221
+ - One primary keyword per line
222
+ - Use the pipe symbol `|` to separate primary keyword from synonyms
223
+ - Use semicolons `;` to separate multiple synonyms
224
+ - Case-insensitive matching (except for special cases like "US")
225
+
226
+ **Special Handling:**
227
+ - "US" is matched exactly to avoid confusion with the word "us"
228
+ - Word boundaries are respected (prevents partial matches)
229
+ - Results are alphabetized and deduplicated
230
+
231
+ **Example Dictionary Entry:**
232
+ ```
233
+ Prisoner of War | Prisoner of War; POW; POWs; prisoner of war
234
+ ```
235
+
236
+ This will find any occurrence of "Prisoner of War", "POW", "POWs", or "prisoner of war" in your text and return all the associated terms.
237
  """)
238
 
239
+ # Footer
240
  gr.HTML("""
241
+ <div style="margin-top: 40px; padding: 20px; background-color: #f8f9fa; border-radius: 8px; text-align: center;">
242
+ <p style="margin: 0; color: #666;">
243
+ Created for digital humanities keyword tagging workflows.
244
+ Based on controlled vocabulary matching principles.
245
+ </p>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  </div>
247
+ """)
248
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
  return demo
250
 
251
  if __name__ == "__main__":