File size: 26,316 Bytes
d62ad66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f259e38
 
 
 
 
 
 
 
 
d62ad66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
import streamlit as st
import pandas as pd
from matplotlib import pyplot as plt
import stanza
from stanza import DownloadMethod

# Load data
@st.cache_data
def load_data():
    df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2')
    return df

# Notification of loading
start_container = st.empty()
if 'first' not in st.session_state:
    st.session_state['first'] = True
    
if st.session_state['first'] == True:
    start_container.text('Loading model and data... This may take two to three minutes on the initial run.')
    st.session_state['first'] = False

# Load the model and data
df = load_data()

# Remove loading notification
start_container.empty()

@st.cache_resource
def load_nlp():
  # Download and initialize a genia pipeline with an anatem NER model
  nlp = stanza.Pipeline('en', package='genia', processors=['tokenize', 'pos', 'lemma'], 
                        use_gpu=True, ner_batch_size=128, tokenize_batch_size=128, 
                        download_method=DownloadMethod.REUSE_RESOURCES, logging_level='WARN')
  return nlp

###### Helper Functions ######
def filter_by_year(data, year_start, year_end):
    
    if year_start == None and year_end == None: 
        return data
    elif year_start == None:
        return data[data['DP'] <= year_end]
    elif year_end == None:  
        return data[data['DP'] >= year_start]

    else: 
        return data.loc[(data['DP'] >= year_start) & (data['DP'] <= year_end)]

def filter_tuples_by_entity_type(data, entity_type):
    '''Filters out tuples in the NER column based on the specified entity type.'''
    
    if entity_type == 'ALL': # Return all data
        return data
    
    data['NER'] = data['NER'].apply(lambda ner_list: [tup for tup in ner_list if tup[1] == entity_type]) # Return only tuples with the specified entity type
    
    return data

def filter_by_country(data, country):
    data = data[data['PL'] == country]
    return data

def group_entity_by_year(data):

    data_explode = data.explode('NER')
    data_explode['NER'] = data_explode['NER'].apply(lambda x: x[0] if not type(x)==float else None) # Return none if the entity is NaN (which is seen as a float)
    data_explode = data_explode.dropna(subset=['NER']) # Drop rows with None as the entity
    
    records = zip(data_explode['DP'], data_explode['NER'])
    
    entitiesDict = {} # {NER: [{year: count}, total_count]

    for yr, name in records:
        if name not in entitiesDict: 
            entitiesDict[name] = [{}, 0]
        entityDict = entitiesDict[name][0]
        if yr not in entityDict: 
            entityDict[yr] = 0
        entityDict[yr] += 1
        entitiesDict[name][1] += 1

    return entitiesDict

def group_journal_by_year(data):

    records = zip(data['DP'], data['JID'], data['JT'])
    journalsDict = {} # {JID: [{DP: count}, total_count, JT]

    for yr, jid, title in records:
        if jid not in journalsDict: 
            journalsDict[jid] = [{}, 0, title]
        journalDict = journalsDict[jid][0]
        if yr not in journalDict: 
            journalDict[yr] = 0
        journalDict[yr] += 1
        journalsDict[jid][1] += 1

    return journalsDict
        
def group_mesh_by_year(data):
    data_explode = data.explode('MH')
    data_explode = data_explode.dropna(subset=['MH']) # Drop rows with NaN as the entity
    records = zip(data_explode['DP'], data_explode['MH'])

    meshesDict = {}
    for yr, word in records:
        # Add mesh term if it does not exist in the dictionary
        if word not in meshesDict: 
            meshesDict[word] = [{}, 0] # Create new entry prepared for the year count. {word: [{year: count}, total_count]}
        
        # Update the year count
        single_meshDict = meshesDict[word][0] # Access the subdictionary in each entry 
        
        if yr not in single_meshDict: 
            single_meshDict[yr] = 0
        single_meshDict[yr] += 1
        meshesDict[word][1] += 1 # Update the total count
    
    return meshesDict

# def find_occurrence(list, target): # Finds the index of the first occurrence of an element in a list which contains tuples
#     index = 0
#     for element in list:
#         if element[0] == target:
#             return index
#         index += 1

###### Collocation Functions ######
def find_collocation(word, texts, n):
# Could work on allowing user to enter list of words.
# Could remove small strings (e.g. "'s") from ngrams. 
# 'America' and 'American' are different words, but 'America' and 'Americans' lemmatize to different words, which is somewhat unexpected behavior to me.
  '''
  Finds neighboring words/n-grams of given word in text.
    
    Parameters:
      word (str): Word to find n-grams of.
      texts (list): List of strings to search for n-grams.
      n (int): Number of words to include in n-gram.
    
    Returns:
      (dict): Dictionary of n-grams with word as key and count as value.
  '''
  # Find n-grams and create dictionary of n-grams
  ngrams = {}
  for text in texts:
    if word in text:
      i = text.index(word)
      for j in range(1, n+1):  # Get required number of words before and after word based on n-gram count specified
        if i-j >= 0:
          if text[i-j] in ngrams:
            ngrams[text[i-j]] += 1
          else:
            ngrams[text[i-j]] = 1
        try:
          if text[i+j] in ngrams:
            ngrams[text[i+j]] += 1
          else:
            ngrams[text[i+j]] = 1
        except IndexError:
          pass
  return dict(sorted(ngrams.items(), key=lambda x:x[1], reverse=True))

# def create_plot(dictionary, word, number_of_items=10):
#   '''Creates a bar chart of the frequency of the items in the dictionary'''
  
#   fig, ax = plt.subplots()
#   ax.bar(x=list(dictionary.keys())[:number_of_items], height=list(dictionary.values())[:number_of_items], align='center')
#   # ax.set_xticklabels(labels=list(dictionary.keys())[:number_of_items], rotation=45)
#   ax.tick_params(axis='x', rotation=45) # Rotates x-axis labels
#   ax.set_xlabel('Words')
#   ax.set_ylabel('Frequency')
#   ax.set_title('Frequency of Words Surrounding ' + '"' + word + '"')
#   return fig
  
def run_collocation(word, degree, dataframe=df, number_of_items=10):
    nlp = load_nlp()
    
    # Lemmatize input word
    process_word = nlp(word)
    lemma = process_word.sentences[0].words[0].lemma
    
    # Find ngrams
    ngrams = find_collocation(word=lemma, texts=dataframe['lemma_AB'], n=degree)
    
    # Plot ngrams
    fig, ax = plt.subplots()
    ax.bar(x=list(ngrams.keys())[:number_of_items], height=list(ngrams.values())[:number_of_items], align='center')
    # ax.set_xticklabels(labels=list(dictionary.keys())[:number_of_items], rotation=45)
    ax.tick_params(axis='x', rotation=45) # Rotates x-axis labels
    ax.set_xlabel('Words')
    ax.set_ylabel('Frequency')
    ax.set_title('Frequency of Words Surrounding ' + '"' + word + '"')
    return fig
    # return create_plot(ngrams, word)

###### Journal Analytics Functions ######
def jid_frequency(data=df, year_start=1975, year_end=2000, n=15):
    '''
    Returns the frequency of the top n most common journal IDs in a given year range.
    '''
    jid_dict = {}
    added = set()  # List used to keep track of all of the items which have already been added to the jid_dict

    dates = data['DP'].tolist()
    jid = data['JID'].tolist()
    index = 0
    for item in jid:
        if dates[index] >= year_start and dates[index] <= year_end:
                if item in added:  # jid is already in the dictionary
                    jid_dict[item] += 1
                else:
                    jid_dict[item] = 1
                    added.add(item)
        index += 1

    # display(jid_dict)  # Used for testing and debugging
    jid_df = pd.DataFrame.from_dict(jid_dict, orient='index', columns=['Frequency'])
    jid_df.sort_values(by=['Frequency'], inplace=True, ascending=False)
    jid_df = jid_df.head(n)

    # Create matplotlib figure from dataframe
    fig, ax = plt.subplots()
    ax.barh(y=jid_df.index, width=jid_df['Frequency'], align='center')
    ax.bar_label(ax.containers[0], fmt='%d', label_type='edge')
    ax.set_xlim(right=jid_df['Frequency'].max() + jid_df['Frequency'].max()*0.08) # Prevents labels from overlapping with border in most cases; increase the number if needed
    ax.set_title('Top {} Most Common JID\'s of Publications'.format(n))
    ax.set_xlabel('Number of publications')
    ax.set_ylabel('Journal NlmId/JID')
    return fig
    # jid_df.plot(kind='barh', figsize=(10, 10), 
    #             title='Top {} Most Common JID\'s of Publications'.format(n), 
    #             legend=True,
    #             xlabel='Number of publications',
    #             ylabel='Journal NlmId/JID',
    #             )
    # # display(jid_df)
    # return(jid_df)

def jid_frequency_by_year(data=df, year_start=1975, year_end=2000, n=15):
    '''
    Creates a line graph showing the frequency at which the most common n JIDs appear in the dataset for each year in given year range.

    Parameters
        data: the medline dataframe
        n: int. The number of journals to display.
        year_start: int. The start of the year range to display. Years for each data point are taken from the 'DP' column.
        year_end: int. The end of the year range to display. Can be the same as year_start.

    Returns
        Matplotib figure object.
    '''

    filtered_by_year = filter_by_year(data, year_start, year_end)
    journalsDict = group_journal_by_year(filtered_by_year)
    sortedJournals = sorted(journalsDict.items(), key = lambda x: x[1][1], reverse = True)
    topJournals = sortedJournals[0:n]
    
    df = pd.DataFrame()

    # Convert the dictionary to a Series and assign it to a new column in the DataFrame
    for jid, data in topJournals:
        yearly_data = data[0]
        # title = data[2]
        df[jid] = pd.Series(yearly_data)

    # Fill NaN values with 0, in case some entities don't have data for all years
    df = df.fillna(0)

    # Debug
    # display(df)

    # Create a figure and axes
    fig, ax = plt.subplots()

    # Plot the data on the axes objects
    for column in df.columns:
        x, y = zip(*sorted(zip(df.index, df[column].values))) # Must sort the data before plotting it; https://stackoverflow.com/questions/37414916/matplotlib-plotting-in-wrong-order
        ax.plot(x, y, marker='o', label=column)
        # ax.plot(df.index, df[column].values)

    # Set plot details
    ax.set_title("Journal Frequencies by Year")
    ax.set_xlabel("Year")
    ax.set_ylabel("Frequency")
    ax.grid(True)
    ax.legend()

    # ax.set_xticks([tick for tick in ax.get_xticks() if tick.is_integer()])
    # ax.set_yticks([tick for tick in ax.get_yticks() if tick.is_integer()])

    return fig

def pl_frequency(data=df, year_start=1975, year_end=2000, n=15):
  '''
  Returns the frequency of the top n most common places of publication in a given year range.
  '''
  pl_dict = {}
  added = set()  # List used to keep track of all of the items which have already been added to the ner_dict

  dates = data['DP'].tolist()
  pl = data['PL'].tolist()
  # display(pl)
  index = 0
  for item in pl:
      if dates[index] >= year_start and dates[index] <= year_end:
          if item in added:  # pl is already in the dictionary
              pl_dict[item] += 1
          else:
              pl_dict[item] = 1
              added.add(item)
      index += 1

  #display(pl_dict)  # Used for testing and debugging
  pl_df = pd.DataFrame.from_dict(
      pl_dict, orient='index', columns=['Frequency'])
  pl_df.sort_values(by=['Frequency'], inplace=True, ascending=False)
  pl_df = pl_df.head(n)

  # Create matplotlib figure from dataframe
  fig, ax = plt.subplots()
  ax.barh(y=pl_df.index, width=pl_df['Frequency'], align='center')
  ax.bar_label(ax.containers[0], fmt='%d', label_type='edge')
  ax.set_xlim(right=pl_df['Frequency'].max() + pl_df['Frequency'].max()*0.10) # Prevents labels from overlapping with border in most cases; increase the number if needed
  ax.set_title('Top {} Most Common Places of Publications'.format(n))
  ax.set_xlabel('Number of publications')
  ax.set_ylabel('Journal NlmId/JID')
  return fig

###### Named Biological Entities Functions ######
def ner_frequency(data=df, n=15, entity_toggle='ALL', year_start=1975, year_end=2000):
    """
    Displays the frequency of the top n most common named entities in a graph.

    Parameters
        data: the medline dataframe (or a subset of it)
        n: int. The number of entities to display.
        entity_toggle: str. The type of entity to display. This info is stored in the tuples in the 'ner' column.
            Options are 'ALL', 'ANATOMY', 'DISEASE'. If 'ALL', displays all entities.
        year_start: int. The start of the year range to display.
        year_end: int. The end of the year range to display. Can be the same as year_start.

    """

    # Function can be slow. Could be improved by using group_entity_by_year() then counting the number of times each year appears.
    ner_dict = {}
    added = set()  # List used to keep track of all of the items which have already been added to the ner_dict

    dates = data['DP'].tolist()
    ner = data['NER'].tolist()

    index = 0
    for item in ner:
        if dates[index] >= year_start and dates[index] <= year_end:
            for tup in item:
                if entity_toggle == 'ALL' or entity_toggle == tup[1]:
                    ner = tup[0]
                    if ner in added:  # ner is already in the dictionary
                        ner_dict[ner] += 1
                    else:
                        ner_dict[ner] = 1
                        added.add(ner)
        index += 1

    # display(ner_dict)  # Used for testing and debugging
    ner_df = pd.DataFrame.from_dict(
        ner_dict, orient='index', columns=['Frequency'])
    ner_df.sort_values(by=['Frequency'], inplace=True, ascending=False)
    ner_df = ner_df.head(n)

    fig, ax = plt.subplots()
    ax.barh(y=ner_df.index, width=ner_df['Frequency'], align='center')
    ax.bar_label(ax.containers[0], fmt='%d', label_type='edge')
    ax.set_xlim(right=ner_df['Frequency'].max() + ner_df['Frequency'].max()*0.08) # Prevents labels from overlapping with border in most cases; increase the number if needed
    if entity_toggle == 'ALL':
        ax.set_title('Top {} Most Common Named Biological Entities'.format(n))
    else:
        ax.set_title('Top {} Most Common Named {} Entities'.format(n, entity_toggle.lower().capitalize()))
    ax.set_xlabel('Number of publications')
    ax.set_ylabel('Journal NlmId/JID')
    return fig

def ner_frequency_by_year(data=df, n=15, entity_toggle='ALL', year_start=1975, year_end=2000) :
    """
    Creates a line graph showing the frequency at which the most common n entities appear in the dataset for each year in given year range.
    In other words, X axis = year, Y axis = frequency, and each line represents a different entity.

    Parameters
        data: the medline dataframe
        n: int. The number of entities to display.
        entity_toggle: str. The type of entity to display. This info is stored in the tuples in the 'NER' column.
            Options are 'ALL', 'ANATOMY', 'DISEASE'. If 'ALL', displays all entities.
        year_start: int. The start of the year range to display. Years for each data point are taken from the 'DP' column.
        year_end: int. The end of the year range to display. Can be the same as year_start.
    
    Returns
        Matplotib figure object.
    """

    filtered_by_year = filter_by_year(data, year_start, year_end)
    filtered_by_entity = filter_tuples_by_entity_type(filtered_by_year, entity_toggle)
    entitiesDict = group_entity_by_year(filtered_by_entity)
    sortedEntities = sorted(entitiesDict.items(), key = lambda x: x[1][1], reverse = True)
    topEntities = sortedEntities[0:n]

    df = pd.DataFrame()

    print(topEntities)
    for entity, data in topEntities:
        # Convert the dictionary to a Series and assign it to a new column in the DataFrame
        yearly_data = data[0]
        df[entity] = pd.Series(yearly_data)

    # Fill NaN values with 0, in case some entities don't have data for all years
    df = df.fillna(0)

    print(df)

    fig, ax = plt.subplots()

    # Plot the data on the axes object
    for column in df.columns:
        x, y = zip(*sorted(zip(df.index, df[column].values))) # Must sort the data before plotting it; https://stackoverflow.com/questions/37414916/matplotlib-plotting-in-wrong-order
        ax.plot(x, y, marker='o', label=column)

    # Set plot details
    ax.set_title("Entity Frequencies by Year")
    ax.set_xlabel("Year")
    ax.set_ylabel("Frequency")
    ax.grid(True)
    ax.legend(loc='upper left')

    return fig

###### MeSH Terms Functions ######
def mh_frequency(country, globally=False, data=df, year_start=1975, year_end=2000, n=15):
    '''
    Returns the frequency of the top n most common MeSH terms in a given year range.
    '''

    mh_dict = {}
    added = set()  # List used to keep track of all of the items which have already been added to the mh_dict

    if globally:
       pass
    else:
       data = filter_by_country(data, country)
    
    dates = data['DP'].tolist()
    mh = data['MH'].tolist()
    index = 0
    for item in mh:
        if dates[index] >= year_start and dates[index] <= year_end:
            for i in item:
                if i in added:  # mh is already in the dictionary
                    mh_dict[i] += 1
                else:
                    mh_dict[i] = 1
                    added.add(i)
        index += 1

    #display(mh_dict)  # Used for testing and debugging
    mh_df = pd.DataFrame.from_dict(
        mh_dict, orient='index', columns=['Frequency'])
    mh_df.sort_values(by=['Frequency'], inplace=True, ascending=False)
    mh_df = mh_df.head(n)

    fig, ax = plt.subplots()
    ax.barh(y=mh_df.index, width=mh_df['Frequency'], align='center')
    ax.bar_label(ax.containers[0], fmt='%d', label_type='edge')
    ax.set_xlim(right=mh_df['Frequency'].max() + mh_df['Frequency'].max()*0.08) # Prevents labels from overlapping with border in most cases; increase the number if needed
    if globally:
        ax.set_title('Top {} Most Common MeSH Terms Globally'.format(n))
    else:
       ax.set_title('Top {} Most Common MeSH Terms in {}'.format(n, country))
    ax.set_xlabel('Number of occurrences')
    ax.set_ylabel('Term')
    return fig

    # mh_df.plot(kind='barh', figsize=(10, 10), title='Top ' +
    #             str(n) + ' Most Common Places of Publication', legend=False)
    # #display(mh_df)
    # return(mh_df)

def mh_time_series(country, globally=False, data=df, year_start=1975, year_end=2000, n=15):
    """
    Creates a matplotlib line plot showing the frequency of the top n most common MeSH topics in a given country (or globally) for each year in a given range.

    The MH column contains MeSH terms (controlled library of words used to categorize articles). 
    The PL column contains publication locations.
    
    Parameters
        country: str. The country to display.
        n: int. The number of MESH topics to display.
        year_start: int. The start of the year range to display. Years for each data point are taken from the 'DP' column.
        year_end: int. The end of the year range to display. Can be the same as year_start.
    
    Returns
        Matplotib figure object.

    # Stores the data in a dictionary with this format:
    pl_mh_dict = {} # {year: {(pl, mh): (count)}} Storing the pl is redundant, but I'd like to keep it for now in case of future changes
    """
    filtered_by_year = filter_by_year(data, year_start, year_end)

    ### TODO: Check if this is fixed
    if globally:
        filtered_by_country = filtered_by_year
    else:
       filtered_by_country = filter_by_country(filtered_by_year, country)
    
    meshesDict = group_mesh_by_year(filtered_by_country)
    sortedMeshes = sorted(meshesDict.items(), key = lambda x: x[1][1], reverse = True)
    topMeshes = sortedMeshes[0:n]


    df = pd.DataFrame()
    for mesh, data in topMeshes:
        # Convert the dictionary to a Series and assign it to a new column in the DataFrame
        yearly_data = data[0]
        
        df[mesh] = pd.Series(yearly_data)

    # Fill NaN values with 0, in case some entities don't have data for all years
    df = df.fillna(0)

    fig, ax = plt.subplots()

    # Plot the data on the axes object
    for column in df.columns:
        x, y = zip(*sorted(zip(df.index, df[column].values)))
        ax.plot(x, y, marker='o', label=column)
        # ax.plot(df.index, df[column], marker='o', label=column)

    # Set plot details
    if globally:
        ax.set_title("MeSH Topic Frequencies by Year Globally")
    else:
       ax.set_title("MeSH Topic Frequencies by Year in " + country)
    ax.set_xlabel("Year")
    ax.set_ylabel("Frequency")
    ax.grid(True)
    ax.legend()

    return fig

###### User Interface ######
with st.sidebar:

  with st.expander('Word Collocation'):
    with st.form(key='collocation_form'):
      word = st.text_input(label='Enter a word to see the most common surrounding words', 
                           placeholder='Enter a keyword')
      degree = st.selectbox('Select the number of adjacent words', [1, 2, 3],
                            help='''If you select 1, the output will include words immediately before and after the keyword. 
                            If you select 2, the output will include words two words before and after the keyword, etc.''')
      # Could also provide an option to override the lemmatization here
      submit_collocation = st.form_submit_button('Submit')

  with st.expander('Journal Analytics'):
     with st.form(key='journal_form'):
      year_start_journal, year_end_journal = st.slider(label='Select a year range', min_value=1975, max_value=2000, value=(1975, 2000), step=1)
      n_journal = st.number_input(label='Enter the number of journals to display', min_value=5, max_value=25, value=10, step=1)
      submit_journal = st.form_submit_button('Submit')

  with st.expander('Named Biological Entities'):
    with st.form(key='nbe_form'):
      entity = st.selectbox('Select an entity type', ['ALL', 'ANATOMY', 'DISEASE'],
                            help='''The natural language processing model that extracts these entities categorizes them as either a disease or an anatomical term. This is an artifact of the corpus that trained the model.''')
      n_nbe = st.number_input(label='Enter the number of entities to display', min_value=5, max_value=25, value=10, step=1)
      year_start_nbe, year_end_nbe = st.slider(label='Select a year range', min_value=1975, max_value=2000, value=(1975, 2000), step=1)

      submit_nbe = st.form_submit_button('Submit')
  
  with st.expander('MeSH Terms'):
     with st.form(key='mesh_form'):
      year_start_mesh, year_end_mesh = st.slider(label='Select a year range', min_value=1975, max_value=2000, value=(1975, 2000), step=1)
      n_mesh = st.number_input(label='Enter the number of MeSH terms to display', min_value=5, max_value=25, value=10, step=1)
      country_mesh = st.selectbox(label='Select a country to view the data related to that location',
                                  help='The names shown here are drawn directly from data attached to the articles.',
                                  options=sorted(df['PL'].unique().tolist()))
      submit_mesh = st.form_submit_button('Submit')

if submit_collocation:
  st.subheader('Collocation')
  st.markdown('This program attempts to find the stem (or root form) of the word you enter and compare it against the database. This can lead to surprising results.')
  collocation_plot = run_collocation(word, degree=degree, number_of_items=10)
  st.pyplot(collocation_plot)

elif submit_journal:
  '''
  Eliminated JID metrics for now because JID is not actually a unique identifier for journals, despite claims to the contrary (https://www.nlm.nih.gov/bsd/mms/medlineelements.html#jid).
  Did something go wrong in the data collection/cleaning process?
  '''
  # st.subheader('Journal Frequency')
  # st.markdown('The NLM lists the names of journals associated with these identifiers [here](https://ftp.ncbi.nih.gov/pubmed/J_Medline.txt).')
  # st.pyplot(jid_frequency(year_start=year_start_journal, year_end=year_end_journal, n=n_journal))
  # st.pyplot(jid_frequency_by_year(year_start=year_start_journal, year_end=year_end_journal, n=n_journal))
  # st.divider()

  st.subheader('Place of Publication Frequency')
  st.markdown('Note that the place of publication is derived from the location of the journal, not the location of the author(s).')
  st.pyplot(pl_frequency(year_start=year_start_journal, year_end=year_end_journal, n=n_journal))

elif submit_nbe:
  st.subheader('Named Biological Entities')
  st.markdown('This program uses a natural language processing model to extract named biological entities from the abstracts in the database. The model categorizes these entities as either a disease or an anatomical term. This is an artifact of the corpus that trained the model.')
  st.pyplot(ner_frequency(n=n_nbe, entity_toggle=entity, year_start=year_start_nbe, year_end=year_end_nbe))
  st.pyplot(ner_frequency_by_year(n=n_nbe, entity_toggle=entity, year_start=year_start_nbe, year_end=year_end_nbe))

elif submit_mesh:
  st.header('MeSH Term Data')
  st.markdown('MeSH (Medical Subject Headings) is the NLM controlled vocabulary thesaurus used for indexing articles. For more information, visit the [MeSH homepage](https://www.nlm.nih.gov/mesh/meshhome.html).')

  st.subheader('MeSH Terms by Country')
  st.pyplot(mh_frequency(country=country_mesh, globally=False, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh))
  st.pyplot(mh_time_series(country=country_mesh, globally=False, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh))

  st.divider()
  
  st.subheader('Mesh Terms Globally')
  st.pyplot(mh_frequency(country=None, globally=True, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh))
  st.pyplot(mh_time_series(country=None, globally=True, data=df, year_start=year_start_mesh, year_end=year_end_mesh, n=n_mesh))