SpEL files required to run the streamlit app copied from https://github.com/shavarani/SpEL

aida.py

@@ -0,0 +1,220 @@
+import os
+import zipfile
+from configuration import get_aida_yago_tsv_file_path, get_resources_dir
+
+TRAIN_START_LINE = "-DOCSTART- (1 EU)"
+TESTA_START_LINE = "-DOCSTART- (947testa CRICKET)"
+TESTB_START_LINE = "-DOCSTART- (1163testb SOCCER)"
+
+CANONICAL_REDIRECTS = None
+
+
+class AnnotationRecord:
+    def __init__(self, line):
+        """
+        Lines with tabs are tokens the are part of a mention:
+            - column 1 is the token
+            - column 2 is either B (beginning of a mention) or I (continuation of a mention)
+            - column 3 is the full mention used to find entity candidates
+            - column 4 is the corresponding YAGO2 entity (in YAGO encoding, i.e. unicode characters are backslash encoded and spaces are replaced by underscores, see also the tools on the YAGO2 website), OR --NME--, denoting that there is no matching entity in YAGO2 for this particular mention, or that we are missing the connection between the mention string and the YAGO2 entity.
+            - column 5 is the corresponding Wikipedia URL of the entity (added for convenience when evaluating against a Wikipedia based method)
+            - column 6 is the corresponding Wikipedia ID of the entity (added for convenience when evaluating against a Wikipedia based method - the ID refers to the dump used for annotation, 2010-08-17)
+            - column 7 is the corresponding Freebase mid, if there is one (thanks to Massimiliano Ciaramita from Google Zürich for creating the mapping and making it available to us)
+        """
+        data_columns = line.split('\t')
+        self.token = None
+        self.begin_inside_tag = None
+        self.full_mention = None
+        self.yago_entity = None
+        self.wikipedia_url = None
+        self.wikipedia_id = None
+        self.freebase_mid = None
+        self.candidates = None
+        if data_columns:
+            self.token = data_columns[0]
+        if len(data_columns) > 1:
+            self.begin_inside_tag = data_columns[1]
+        if len(data_columns) > 2:
+            self.full_mention = data_columns[2]
+        if len(data_columns) > 3:
+            self.yago_entity = data_columns[3]
+        if len(data_columns) > 4:
+            self.wikipedia_url = data_columns[4]
+        if len(data_columns) > 5:
+            self.wikipedia_id = data_columns[5]
+        if len(data_columns) > 6:
+            self.freebase_mid = data_columns[6]
+
+    def set_candidates(self, candidate_record):
+        self.candidates = candidate_record
+        self.candidates.non_considered_word_count -= 1
+
+    def __str__(self):
+        res = ""
+        t = [self.token, self.begin_inside_tag, self.full_mention, self.yago_entity, self.wikipedia_url,
+             self.wikipedia_id, self.freebase_mid]
+        for ind, e in enumerate(t):
+            if not e:
+                continue
+            if ind < len(t) - 1:
+                res += e + "|"
+            else:
+                res += e
+        if res[-1] == "|":
+            res = res[:-1]
+        return res
+
+
+class Document:
+    def __init__(self, document_id):
+        self.document_id = document_id
+        self.annotations = []
+        self.current_annotation = []
+
+    def add_annotation(self, line, candidates):
+        if not line:
+            self.flush_current_annotation()
+        else:
+            ar = AnnotationRecord(line)
+            for c in candidates:
+                if c.non_considered_word_count < 1:
+                    continue
+                if c.orig_text == ar.full_mention:
+                    ar.set_candidates(c)
+                    break
+            self.current_annotation.append(ar)
+
+    def flush_current_annotation(self):
+        self.annotations.append(self.current_annotation)
+        self.current_annotation = []
+
+
+class Candidate:
+    def __init__(self, candidate_line):
+        self.id = ""
+        self.in_count = 0
+        self.out_count = 0
+        self.links = 0
+        self.url = ""
+        self.name = ""
+        self.normal_name = ""
+        self.normal_wiki_title = ""
+        self.predicted_type = ""
+        for item in candidate_line.split('\t'):
+            if item == 'CANDIDATE' or not item.strip():
+                continue
+            elif item.startswith('id:'):
+                self.id = item[3:]
+            elif item.startswith('inCount:'):
+                self.in_count = int(item[8:])
+            elif item.startswith('outCount:'):
+                self.out_count = int(item[9:])
+            elif item.startswith('links:'):
+                self.links = item[6:]
+            elif item.startswith('url:'):
+                self.url = item[4:]
+            elif item.startswith('name:'):
+                self.name = item[5:]
+            elif item.startswith('normalName:'):
+                self.normal_name = item[11:]
+            elif item.startswith('normalWikiTitle:'):
+                self.normal_wiki_title = item[16:]
+            elif item.startswith('predictedType:'):
+                self.predicted_type = item[14:]
+            else:
+                raise ValueError(f"Undefined PPRforNED CANDIDATE column: {item}")
+
+    def __str__(self):
+        return f"id: {self.id}\twiki_page: {self.url.replace('http://en.wikipedia.org/wiki/', '')}"
+
+
+class CandidateRecord:
+    def __init__(self, entity_header):
+        self.candidates = []
+        self.text = ""
+        self.normal_name = ""
+        self.predicted_type = ""
+        self.q = False
+        self.qid = ""
+        self.docid = -1
+        self.orig_text = ""
+        self.non_considered_word_count = 0
+        self.url = ""
+        for item in entity_header.split('\t'):
+            if item == 'ENTITY':
+                continue
+            elif item.startswith('text:'):
+                self.text = item[5:]
+            elif item.startswith('normalName:'):
+                self.normal_name = item[11:]
+            elif item.startswith('predictedType:'):
+                self.predicted_type = item[14:]
+            elif item.startswith('q:'):
+                self.q = bool(item[2:])
+            elif item.startswith('qid:'):
+                self.qid = item[4:]
+            elif item.startswith('docId:'):
+                self.docid = int(item[6:]) - 1
+            elif item.startswith('origText:'):
+                self.orig_text = item[9:]
+                self.non_considered_word_count = len(self.orig_text.split())
+            elif item.startswith('url:'):
+                self.url = item[4:]
+            else:
+                raise ValueError(f"Undefined PPRforNED column: {item}")
+
+    def add_candidate(self, candidate_line):
+        self.candidates.append(Candidate(candidate_line))
+
+    def __str__(self):
+        cnds = '\n\t'.join([str(x) for x in self.candidates])
+        return f"doc_id: {self.docid}\toriginal_text: {self.orig_text}\tcandidates:\n\t{cnds}"
+
+
+def get_candidates(ppr_for_ned_candidates_zip, last_document_id):
+    candidates_string = ppr_for_ned_candidates_zip.read(str(last_document_id + 1)).decode("utf-8").split("\n")
+    candidates = []
+    for c_line in candidates_string:
+        if not c_line.strip():
+            continue
+        if c_line.startswith("ENTITY"):
+            candidates.append(CandidateRecord(c_line))
+        elif c_line.startswith("CANDIDATE"):
+            assert len(candidates)
+            candidates[-1].add_candidate(c_line)
+        else:
+            raise ValueError("This must be unreachable!")
+    return candidates
+
+class AIDADataset:
+    def __init__(self):
+        super(AIDADataset, self).__init__()
+        self.dataset = None
+        self.data_path = str(get_aida_yago_tsv_file_path().absolute())
+        assert os.path.exists(self.data_path), f"The passed dataset address: {self.data_path} does not exist"
+        self.load_dataset()
+
+    def load_dataset(self):
+        ppr_for_ned_candidates_zip = zipfile.ZipFile(get_resources_dir() / "data" / "PPRforNED.zip", "r")
+        annotations = [[], [], []]
+        current_document = None
+        current_document_candidates = None
+        data_split_id = -1
+        last_document_id = 0
+        with open(self.data_path, "r", encoding="utf-8") as data_file:
+            for ind, line in enumerate(data_file):
+                line = line.strip()
+                if line.startswith("-DOCSTART-"):
+                    if current_document:
+                        annotations[data_split_id].append(current_document)
+                        last_document_id += 1
+                    if line == TRAIN_START_LINE or line == TESTA_START_LINE or line == TESTB_START_LINE:
+                        data_split_id += 1
+                    current_document = Document(last_document_id)
+                    current_document_candidates = get_candidates(ppr_for_ned_candidates_zip, last_document_id)
+                else:
+                    current_document.add_annotation(line, current_document_candidates)
+            if current_document:
+                annotations[data_split_id].append(current_document)
+        self.dataset = {"train": annotations[0], "testa": annotations[1], "testb": annotations[2]}
+        ppr_for_ned_candidates_zip.close()

app.py

@@ -0,0 +1,75 @@
+"""
+The SpEL annotation visualization script. You can use this script as a playground to explore the capabilities and
+limitations of the SpEL framework.
+"""
+import torch
+from model import SpELAnnotator
+from data_loader import dl_sa
+from utils import chunk_annotate_and_merge_to_phrase
+from candidate_manager import CandidateManager
+import streamlit as st
+from annotated_text import annotated_text
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+@st.cache_resource
+def load_model():
+    load_aida_finetuned = True
+    load_full_vocabulary=True
+    candidate_setting = "n"
+    model = SpELAnnotator()
+    model.init_model_from_scratch(device=device)
+    candidates_manager_to_use = CandidateManager(dl_sa.mentions_vocab,
+                                                 is_kb_yago=candidate_setting == "k",
+                                                 is_ppr_for_ned=candidate_setting.startswith("p"),
+                                                 is_context_agnostic=candidate_setting == "pg",
+                                                 is_indexed_for_spans=True) if candidate_setting != "n" else None
+    if load_aida_finetuned and not load_full_vocabulary:
+        model.shrink_classification_head_to_aida(device=device)
+        model.load_checkpoint(None, device=device, load_from_torch_hub=True, finetuned_after_step=3)
+    elif load_aida_finetuned:
+        model.load_checkpoint(None, device=device, load_from_torch_hub=True, finetuned_after_step=4)
+    else:
+        model.load_checkpoint(None, device=device, load_from_torch_hub=True, finetuned_after_step=2)
+    return model, candidates_manager_to_use
+
+annotator, candidates_manager = load_model()
+st.title("SpEL Prediction Visualization")
+mention = st.text_input("Enter the text:")
+process_button = st.button("Annotate")
+
+if process_button and mention:
+    phrase_annotations = chunk_annotate_and_merge_to_phrase(
+        annotator, mention, k_for_top_k_to_keep=5, normalize_for_chinese_characters=True)
+    last_step_annotations = [[p.words[0].token_offsets[0][1][0],
+                              p.words[-1].token_offsets[-1][1][-1],
+                              (dl_sa.mentions_itos[p.resolved_annotation], p.subword_annotations)]
+                             for p in phrase_annotations if p.resolved_annotation != 0]
+    if candidates_manager:
+        for p in phrase_annotations:
+            candidates_manager.modify_phrase_annotation_using_candidates(p, mention)
+    if last_step_annotations:
+        anns = sorted([(l_ann[0], l_ann[1], l_ann[2][0]) for l_ann in last_step_annotations], key=lambda x: x[0])
+        begin = 0
+        last_char = len(mention)
+        anns_pointer = 0
+        processed_anns = []
+        anno_text = []
+        while begin < last_char:
+            if anns_pointer == len(anns):
+                processed_anns.append((begin, last_char, "O"))
+                anno_text.append(mention[begin: last_char])
+                begin = last_char
+                continue
+            first_unprocessed_annotation = anns[anns_pointer]
+            if first_unprocessed_annotation[0] > begin:
+                processed_anns.append((begin, first_unprocessed_annotation[0], "O"))
+                anno_text.append(mention[begin: first_unprocessed_annotation[0]])
+                begin = first_unprocessed_annotation[0]
+            else:
+                processed_anns.append(first_unprocessed_annotation)
+                anns_pointer += 1
+                begin = first_unprocessed_annotation[1]
+                anno_text.append((mention[first_unprocessed_annotation[0]: first_unprocessed_annotation[1]], first_unprocessed_annotation[2]))
+        annotated_text(anno_text)
+    else:
+        annotated_text(mention)

base_model.cfg

@@ -0,0 +1 @@
+roberta-base

candidate_manager.py

@@ -0,0 +1,103 @@
+"""
+This file contains the implementation of the candidate manager in charge of loading the candidate sets,
+ and modifying the phrase annotations using the loaded candidates.
+"""
+import json
+from span_annotation import PhraseAnnotation
+from configuration import get_resources_dir
+
+
+class CandidateManager:
+    def __init__(self, mentions_vocab, is_kb_yago = False, is_ppr_for_ned = False, is_context_agnostic = False,
+                 is_indexed_for_spans= False):
+        self.mentions_vocab = mentions_vocab
+        self.candidates = None
+        if is_kb_yago:
+            print(" * Loading the candidates stored for KB+YAGO ...")
+            is_context_agnostic = True
+            is_indexed_for_spans = False
+            self.load_kb_plus_yago()
+        elif is_ppr_for_ned:
+            print(" * Loading the {} PPRforNED candidate set ...".format(
+                'context agnostic' if is_context_agnostic else 'context aware'))
+            self.load_ppr_for_ned_candidates(is_context_agnostic, is_indexed_for_spans)
+        else:
+            raise ValueError("Either \'is_kb_yago\' or \'is_ppr_for_ned\' flags must be True!")
+        self.is_context_agnostic = is_context_agnostic
+        self.is_indexed_for_spans = is_indexed_for_spans
+        self.is_kb_yago = is_kb_yago
+        self.is_ppr_for_ned = is_ppr_for_ned
+
+    def load_ppr_for_ned_candidates(self, is_context_agnostic, is_indexed_for_spans):
+        if is_context_agnostic:
+            file_address = "context_agnostic_mentions.json"
+        elif is_indexed_for_spans:
+            file_address = "context_aware_spans.json"
+        else:
+            file_address = "context_aware_mentions.json"
+        candidates_a = json.load(open(
+            get_resources_dir() / "data" / "candidates" / "aida_testa_pprforned" / file_address, "r"))
+        candidates_b = json.load(open(
+            get_resources_dir() / "data" / "candidates" / "aida_testb_pprforned" / file_address, "r"))
+        if is_context_agnostic:
+            for key in candidates_b:
+                if key in candidates_a:
+                    for elem in candidates_b[key]:
+                        if elem not in candidates_a[key]:
+                            candidates_a[key].append(elem)
+                else:
+                    candidates_a[key] = candidates_b[key]
+        else:
+            candidates_a.update(candidates_b)
+        self.candidates = candidates_a
+
+    def load_kb_plus_yago(self):
+        self.candidates = json.load(open(
+            get_resources_dir() / "data" / "candidates" / "kb_plus_yago_candidates.json", "r"))
+
+    def _fetch_candidates(self, phrase_annotation, sentence = None):
+        candidates = []
+        if self.is_kb_yago:
+            phrase_to_check = phrase_annotation.word_string.lower()
+            if phrase_to_check in self.candidates:
+                candidates = self.candidates[phrase_to_check]
+        elif self.is_ppr_for_ned:
+            # TODO lower-cased check mention surface forms
+            span_key  = f"({phrase_annotation.begin_character}, {phrase_annotation.end_character})"
+            if self.is_context_agnostic and phrase_annotation.word_string in self.candidates:
+                candidates = self.candidates[phrase_annotation.word_string]
+            elif not self.is_context_agnostic and sentence in self.candidates:
+                if self.is_indexed_for_spans and span_key in self.candidates[sentence]:
+                    candidates = self.candidates[sentence][span_key]
+                elif not self.is_indexed_for_spans and phrase_annotation.word_string in self.candidates[sentence]:
+                    candidates = self.candidates[sentence][phrase_annotation.word_string]
+        return candidates
+
+    def modify_phrase_annotation_using_candidates(self, phrase_annotation: PhraseAnnotation, sentence: str = None):
+        """
+        The method post processes the :param phrase_annotation: found in a :param sentence: to make sure it is bound to
+          the predefined {self.candidates} set.
+        It is not possible to perform candidate look up for spans in context agnostic scenario
+          so {self.is_indexed_for_spans} will only be considered where {self.is_context_agnostic} is False.
+        """
+        if self.candidates is None or phrase_annotation.resolved_annotation == 0:
+            return
+        candidates = self._fetch_candidates(phrase_annotation, sentence)
+        if not candidates:
+            phrase_annotation.set_alternative_as_resolved_annotation(0)
+            return
+        if self.is_kb_yago:
+            candidates_ = [self.mentions_vocab[x[0]] for x in candidates if x[0] in self.mentions_vocab]
+            prior_probabilities_ = [x[1] for x in candidates if x[0] in self.mentions_vocab]
+        else:
+            candidates_ = [self.mentions_vocab[x] for x in candidates if x in self.mentions_vocab]
+            prior_probabilities_ = [1.0 for x in candidates if x in self.mentions_vocab]
+        # TODO use the prior_probabilities_ to adjust the probabilities
+        if candidates_:
+            all_p_anns = phrase_annotation.all_possible_annotations()
+            filtered_p_predictions = sorted(
+                [x for x in all_p_anns if x[0] in candidates_], key=lambda y: y[1], reverse=True)
+            if filtered_p_predictions:
+                phrase_annotation.set_alternative_as_resolved_annotation(filtered_p_predictions[0][0])
+            else:
+                phrase_annotation.set_alternative_as_resolved_annotation(0)

configuration.py

@@ -0,0 +1,104 @@
+import os
+import torch
+import pathlib
+import json
+from datetime import date
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+AIDA_CANONICAL_REDIRECTS = None
+OOD_CANONICAL_REDIRECTS = None
+
+
+def get_base_model_name():
+    return open("base_model.cfg", "r").read().strip()
+
+
+def get_project_top_dir():
+    return pathlib.Path(os.path.abspath(__file__)).parent
+
+
+def get_resources_dir():
+    return get_project_top_dir() / 'resources'
+
+
+def get_checkpoints_dir():
+    path_ = get_project_top_dir() / '.checkpoints'
+    if not os.path.exists(path_):
+        os.mkdir(path_)
+    return path_
+
+
+def get_logdir_dir():
+    path_ = get_project_top_dir() / '.logdir'
+    if not os.path.exists(path_):
+        os.mkdir(path_)
+    return path_
+
+
+def get_aida_train_canonical_redirects():
+    global AIDA_CANONICAL_REDIRECTS
+    if not AIDA_CANONICAL_REDIRECTS:
+        r_file = get_resources_dir() / "data" / "aida_canonical_redirects.json"
+        with r_file.open() as f:
+            AIDA_CANONICAL_REDIRECTS = json.load(f)
+    return AIDA_CANONICAL_REDIRECTS
+
+def get_ood_canonical_redirects():
+    global OOD_CANONICAL_REDIRECTS
+    if not OOD_CANONICAL_REDIRECTS:
+        r_file = get_resources_dir() / "data" / "ood_canonical_redirects.json"
+        with r_file.open() as f:
+            OOD_CANONICAL_REDIRECTS = json.load(f)
+    return OOD_CANONICAL_REDIRECTS
+
+
+def get_aida_yago_tsv_file_path():
+    return get_resources_dir() / "data" / "AIDA-YAGO2-dataset.tsv"
+
+
+def get_exec_run_file():
+    return get_logdir_dir() / f"annotator_log-{date.today().strftime('%Y-%b-%d')}.log"
+
+
+def get_aida_vocab():
+    mentions_vocab = dict({'|||O|||': 0, '<pad>': 1})
+    dictionary_file = get_resources_dir() / "vocab" / "aida.txt"
+    dfile = dictionary_file.open("r")
+    for _ad_element in dfile.read().split("\n"):
+        mentions_vocab[_ad_element] = len(mentions_vocab)
+    return mentions_vocab
+
+def get_ood_vocab():
+    # This function might be used if one is interested in testing out the "masking all the candidates not in our
+    #   expected entity set" which is mentioned in the footnote of section 4.1 of the paper.
+    mentions_vocab = dict({'|||O|||': 0, '<pad>': 1})
+    dictionary_file = get_resources_dir() / "vocab" / "out_of_domain.txt"
+    dfile = dictionary_file.open("r")
+    for _ad_element in dfile.read().split("\n"):
+        mentions_vocab[_ad_element] = len(mentions_vocab)
+    return mentions_vocab
+
+
+def get_aida_plus_wikipedia_vocab():
+    mentions_vocab = get_aida_vocab()
+    dictionary_file = get_resources_dir() / "vocab" / f"enwiki_20230827.txt"
+    dfile = dictionary_file.open("r")
+    for _ad_element in dfile.read().split("\n"):
+        if _ad_element not in mentions_vocab:
+            mentions_vocab[_ad_element] = len(mentions_vocab)
+    return mentions_vocab
+
+def get_aida_plus_wikipedia_plus_out_of_domain_vocab():
+    mentions_vocab = get_aida_plus_wikipedia_vocab()
+    dictionary_file = get_resources_dir() / "vocab" / f"out_of_domain.txt"
+    dfile = dictionary_file.open("r")
+    for _ad_element in dfile.read().split("\n"):
+        if _ad_element not in mentions_vocab:
+            mentions_vocab[_ad_element] = len(mentions_vocab)
+    return mentions_vocab
+
+def get_n3_entity_to_kb_mappings():
+    kb_file = get_resources_dir() / "data" / "n3_kb_mappings.json"
+    knowledge_base = json.load(kb_file.open("r"))
+    return knowledge_base

data_loader.py

@@ -0,0 +1,390 @@
+"""
+The enwiki/conll Dataset reader/provider using torchtext.
+The datasets were crated using the scripts from:
+    https://github.com/samuelbroscheit/entity_knowledge_in_bert/tree/master/bert_entity/preprocessing
+The get_dataset.collate_batch function is influenced by:
+    https://raw.githubusercontent.com/samuelbroscheit/entity_knowledge_in_bert/master/bert_entity/data_loader_wiki.py
+
+Please note that the pre-processed fine-tuning data will be automatically downloaded upon instantiation of the data
+ readers and the result will be saved under /home/<user_name>/.cache/torch/text/datasets/ (in linux systems)
+
+The expected sizes of the auto-downloaded datasets:
+    - Step 1 (general knowledge fine-tuning):
+            enwiki-2023-spel-roberta-tokenized-aug-27-2023.tar.gz: 19.1 GBs
+            * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+            * You can delete the file above once fine-tuning step 1 is done, and you are moving on to step 2.         *
+            * in the cleaning up process, make sure you remove the cached validation set files under .checkpoints     *
+            * directory as well                                                                                       *
+            * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+    - Step 2 (general knowledge fine-tuning):
+            enwiki-2023-spel-roberta-tokenized-aug-27-2023-retokenized.tar.gz: 17.5 GBs
+            * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+            * You can delete the file above once fine-tuning step 2 is done, and you are moving on to step 3.         *
+            * in the cleaning up process, make sure you remove the cached validation set files under .checkpoints     *
+            * directory as well                                                                                       *
+            * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+    - Step 3 (domain specific fine-tuning):
+            aida-conll-spel-roberta-tokenized-aug-23-2023.tar.gz: 5.1 MBs
+
+No extra preprocessing step will be required, as soon as you start the fine-tuning script for each step,
+ the proper fine-tuning dataset will be downloaded and will be served **without** the need for unzipping.
+"""
+import os
+import json
+import numpy
+from functools import partial
+from collections import OrderedDict
+from tqdm import tqdm
+from typing import Union, Tuple
+
+import torch
+from torch.utils.data import DataLoader, Dataset
+from torch.utils.data.distributed import DistributedSampler
+from torchdata.datapipes.iter import FileOpener, IterableWrapper, HttpReader, FileLister
+from torchtext.data.datasets_utils import _wrap_split_argument, _create_dataset_directory
+from torchtext.utils import download_from_url
+
+from transformers import AutoTokenizer, BatchEncoding
+
+from configuration import (get_aida_plus_wikipedia_plus_out_of_domain_vocab, get_aida_train_canonical_redirects,
+                                get_aida_vocab, get_ood_vocab, get_checkpoints_dir, get_base_model_name, device)
+
+BERT_MODEL_NAME = get_base_model_name()
+MAX_SPAN_ANNOTATION_SIZE = 4
+
+
+class StaticAccess:
+    def __init__(self):
+        self.mentions_vocab, self.mentions_itos = None, None
+        self.set_vocab_and_itos_to_all()
+        self.aida_canonical_redirects = get_aida_train_canonical_redirects()
+        self._all_vocab_mask_for_aida = None
+        self._all_vocab_mask_for_ood = None
+
+    def set_vocab_and_itos_to_all(self):
+        self.mentions_vocab = get_aida_plus_wikipedia_plus_out_of_domain_vocab()
+        self.mentions_itos = [w[0] for w in sorted(self.mentions_vocab.items(), key=lambda x: x[1])]
+
+    @staticmethod
+    def get_aida_vocab_and_itos():
+        aida_mentions_vocab = get_aida_vocab()
+        aida_mentions_itos = [w[0] for w in sorted(aida_mentions_vocab.items(), key=lambda x: x[1])]
+        return aida_mentions_vocab, aida_mentions_itos
+
+    def shrink_vocab_to_aida(self):
+        self.mentions_vocab, self.mentions_itos = self.get_aida_vocab_and_itos()
+
+    def get_all_vocab_mask_for_aida(self):
+        if self._all_vocab_mask_for_aida is None:
+            mentions_vocab = get_aida_plus_wikipedia_plus_out_of_domain_vocab()
+            mask = torch.ones(len(mentions_vocab)).to(device)
+            mask = mask * -10000
+            mask[torch.Tensor([mentions_vocab[x] for x in get_aida_vocab()]).long()] = 0
+            self._all_vocab_mask_for_aida = mask
+        return self._all_vocab_mask_for_aida
+
+    def get_all_vocab_mask_for_ood(self):
+        if self._all_vocab_mask_for_ood is None:
+            mentions_vocab = get_aida_plus_wikipedia_plus_out_of_domain_vocab()
+            mask = torch.ones(len(mentions_vocab)).to(device)
+            mask = mask * -10000
+            mask[torch.Tensor([mentions_vocab[x] for x in get_ood_vocab()]).long()] = 0
+            self._all_vocab_mask_for_ood = mask
+        return self._all_vocab_mask_for_ood
+
+
+dl_sa = StaticAccess()
+
+
+class ENWIKI20230827Config:
+    URL = "https://1sfu-my.sharepoint.com/:u:/g/personal/sshavara_sfu_ca/Ea3IVbOpkTJKpASNyL9aFGMBQpH0ABU2hQa-wYyakkZ9TQ?e=DJFF3v&download=1"
+    MD5 = "eb9a54a8f1f858cdcbf6c750942a896f"
+    PATH = "enwiki-2023-spel-roberta-tokenized-aug-27-2023.tar.gz"
+    DATASET_NAME = "WIKIPEDIA20230827"
+    NUM_LINES = {'train': 3055221, 'valid': 1000, 'test': 1000}
+
+
+class ENWIKI20230827V2Config:
+    URL = 'https://1sfu-my.sharepoint.com/:u:/g/personal/sshavara_sfu_ca/EeS_Tgl_CFJNiTh6YH5IDrsBocEZUsZV3lxPB6pleTxyxw?e=caH1cf&download=1'
+    MD5 = "83a37f528800a463cd1a376e80ffc744"
+    PATH = "enwiki-2023-spel-roberta-tokenized-aug-27-2023-retokenized.tar.gz"
+    DATASET_NAME = "WIKIPEDIA20230827V2"
+    NUM_LINES = {'train': 3038581, 'valid': 996}
+
+
+class AIDA20230827Config:
+    URL = "https://1sfu-my.sharepoint.com/:u:/g/personal/sshavara_sfu_ca/EajEGYyf8LBOoxqDaiPBvbgBwFuEC08nssvZwGJWsG_HXg?e=wAwV6H&download=1"
+    MD5 = "8078529d5df96d0d1ecf6a505fdb767a"
+    PATH = "aida-conll-spel-roberta-tokenized-aug-23-2023.tar.gz"
+    DATASET_NAME = "AIDA20230827"
+    NUM_LINES = {'train': 1585, 'valid': 391, 'test': 372}
+
+
+tokenizer = AutoTokenizer.from_pretrained(BERT_MODEL_NAME, cache_dir=get_checkpoints_dir() / "hf")
+
+WIKI_EXTRACTED_FILES = {"train": "train.json", "valid": "valid.json", "test": " test.json"}
+
+
+def wiki_filter_fn(split, fname_and_stream):
+    return WIKI_EXTRACTED_FILES[split] in fname_and_stream[0]
+
+
+def wiki_data_record_convert(line):
+    element = json.loads(line)
+    r = {'tokens': element['tokens'], 'mentions': [], 'mention_entity_probs': [], 'mention_probs': []}
+    for token, mentions, mention_entity_probs, mention_probs in zip(element['tokens'], element['mentions'],
+                                                                    element['mention_entity_probs'],
+                                                                    element['mention_probs']):
+        if len(mention_probs) < len(mentions):
+            mention_probs.extend([1.0 for _ in range(len(mentions) - len(mention_probs))])
+        sorted_mentions = sorted(list(zip(mentions, mention_entity_probs, mention_probs)),
+                                 key=lambda x: x[1], reverse=True)
+        mentions_ = [dl_sa.aida_canonical_redirects[x[0]] if x[0] in dl_sa.aida_canonical_redirects else x[0]
+                     for x in sorted_mentions if x[0]]  # ignore mentions that are None
+        mention_entity_probs_ = [x[1] for x in sorted_mentions if x[0]]  # ignore prob. for None mentions
+        mention_probs_ = [x[2] for x in sorted_mentions if x[0]]  # ignore m_probs for None mentions
+        r['mentions'].append(mentions_[:MAX_SPAN_ANNOTATION_SIZE])
+        r['mention_probs'].append(mention_probs_[:MAX_SPAN_ANNOTATION_SIZE])
+        r['mention_entity_probs'].append(mention_entity_probs_[:MAX_SPAN_ANNOTATION_SIZE])
+        if len(mentions_) > MAX_SPAN_ANNOTATION_SIZE:
+            r['mention_entity_probs'][-1] = [x / sum(r['mention_entity_probs'][-1])
+                                             for x in r['mention_entity_probs'][-1]]
+    return r
+
+@_create_dataset_directory(dataset_name=ENWIKI20230827Config.DATASET_NAME)
+@_wrap_split_argument(("train", "valid", "test"))
+def ENWIKI20230827(root: str, split: Union[Tuple[str], str]):
+    root = root if root else ".data"
+    path = root + "/" + ENWIKI20230827Config.PATH
+    if not os.path.exists(path):
+        download_from_url(ENWIKI20230827Config.URL, root=root, path=path, hash_value=ENWIKI20230827Config.MD5,
+                          hash_type='md5')
+    online_reader_dp = FileLister(root, ENWIKI20230827Config.PATH)
+    tar_file_dp = FileOpener(online_reader_dp, mode="b").load_from_tar().filter(
+        partial(wiki_filter_fn, split)).readlines(return_path=False).map(wiki_data_record_convert)
+    return tar_file_dp
+
+@_create_dataset_directory(dataset_name=ENWIKI20230827V2Config.DATASET_NAME)
+@_wrap_split_argument(("train", "valid"))
+def ENWIKI20230827V2(root: str, split: Union[Tuple[str], str]):
+    root = root if root else ".data"
+    path = root + "/" + ENWIKI20230827V2Config.PATH
+    if not os.path.exists(path):
+        download_from_url(ENWIKI20230827V2Config.URL, root=root, path=path, hash_value=ENWIKI20230827V2Config.MD5,
+                          hash_type='md5')
+    online_reader_dp = FileLister(root, ENWIKI20230827V2Config.PATH)
+    tar_file_dp = FileOpener(online_reader_dp, mode="b").load_from_tar().filter(
+        partial(wiki_filter_fn, split)).readlines(return_path=False).map(wiki_data_record_convert)
+    return tar_file_dp
+
+def aida_path_fn(r, _=None):
+    return os.path.join(r, AIDA20230827Config.PATH)
+
+
+def aida_select_split(s, file_name_data):
+    return file_name_data[1][s]
+
+
+def aida_data_record_convert(r):
+    for x in r:  # making sure each token comes with exactly one annotation
+        assert len(x) == 7 or len(x) == 8  # whether it contains the candidates or not
+        return {"tokens": [x[0] for x in r], "mentions": [[x[4] if x[4] else "|||O|||"] for x in r],
+                "mention_entity_probs": [[1.0] for _ in r], "mention_probs": [[1.0] for _ in r],
+                "candidates": [x[7] if x[7] else [] for x in r] if len(x) == 8 else [[] for x in r]}
+
+
+@_create_dataset_directory(dataset_name=AIDA20230827Config.DATASET_NAME)
+@_wrap_split_argument(('train', 'valid', 'test'))
+def AIDA20230827(root, split):
+    online_reader_dp = HttpReader(IterableWrapper([AIDA20230827Config.URL])).on_disk_cache(
+        filepath_fn=partial(aida_path_fn, root), hash_dict={aida_path_fn(root): AIDA20230827Config.MD5},
+        hash_type="md5").end_caching(mode="wb", same_filepath_fn=True)
+    return FileOpener(online_reader_dp, mode="b").load_from_tar().parse_json_files().flatmap(
+        partial(aida_select_split, split)).map(aida_data_record_convert)
+
+
+class DistributableDataset(Dataset):
+    """
+    Based on the documentations in torch.utils.data.DataLoader, `IterableDataset` does not support custom `sampler`
+    Therefore we cannot use the DistributedSampler with the DataLoader to split the data samples.
+    This class is a workaround to make the IterableDataset work with the DistributedSampler.
+    """
+    def __init__(self, dataset, size, world_size, rank):
+        self.size = size
+        self.data = iter(dataset)
+        self.world_size = world_size
+        self.rank = rank
+        self.initial_fetch = True
+
+    def __len__(self):
+        return self.size
+
+    def __getitem__(self, index):
+        # Since we don't have a means of accessing the data by indices, we try skipping the indices that we believe
+        #  belong to other processes
+        skip_size = self.rank if self.initial_fetch else self.world_size - 1
+        self.initial_fetch = False
+        for _ in range(skip_size):
+            next(self.data)
+        return next(self.data)
+
+
+def convert_is_in_mention_to_bioes(is_in_mention):
+    # B = 0, I = 1, O = 2, E = 3, S = 4
+    bioes = []
+    for iim, current in enumerate(is_in_mention):
+        before = is_in_mention[iim - 1] if iim > 0 else 0
+        after = is_in_mention[iim + 1] if iim < len(is_in_mention) - 1 else 0
+        bioes.append(
+            2 if not current else (4 if not before and not after else (0 if not before else (3 if not after else 1))))
+    return bioes
+
+
+def get_dataset(dataset_name: str, split: str, batch_size: int, get_labels_with_high_model_score=None,
+                label_size: int = 0, load_distributed: bool = False, world_size: int = 1, rank: int = 0,
+                use_retokenized_wikipedia_data: bool = False):
+    """
+    :param dataset_name: The dataset name can either be "enwiki" or "aida"
+    :param split: the requested dataset split which can be 'train', 'valid' or 'test'
+    :param batch_size: the size of the resulting batch from the data loader
+    :param get_labels_with_high_model_score: The function that finds high scoring negative samples for the model
+    :param label_size: The maximum output distribution size. You can pass the output vocabulary size for this parameter.
+    :param load_distributed: The flag hinting whether the data loader will be loaded in a multi-gpu setting.
+    :param world_size: the number of machines that the dataloader is expected to serve.
+    :param rank: the rank of the gpu on which the data is expected to be served.
+    :param use_retokenized_wikipedia_data: a flag indicating whether to use ENWIKI20230827 dataset or ENWIKI20230827V2
+    """
+
+    assert dataset_name in ["enwiki", "aida"]
+    if not load_distributed or rank == 0:
+        print(f"Initializing the {dataset_name.upper()}/{split} dataset ...")
+
+    def collate_batch(batch):
+        data = {}
+        for key in ["tokens", "mentions", "mention_entity_probs", "eval_mask", "candidates", "is_in_mention", "bioes"]:
+            data[key] = []
+        for annotated_line_in_file in batch:
+            data["tokens"].append(tokenizer.convert_tokens_to_ids(annotated_line_in_file["tokens"]))
+            data["mentions"].append([
+                [(dl_sa.mentions_vocab[x] if x not in dl_sa.aida_canonical_redirects else
+                  dl_sa.mentions_vocab[dl_sa.aida_canonical_redirects[x]])
+                 if x is not None and x not in ['Gmina_Żabno'] else dl_sa.mentions_vocab["|||O|||"] for x in el]
+                for el in annotated_line_in_file["mentions"]
+            ])
+            data["mention_entity_probs"].append(annotated_line_in_file["mention_entity_probs"])
+            data["eval_mask"].append(list(map(
+                lambda item: 1 if len(item) == 1 else 0, annotated_line_in_file["mention_probs"])))
+            is_in_mention = [1 if x != '|||O|||' else 0 for el, elp in zip(
+                annotated_line_in_file["mentions"], annotated_line_in_file["mention_entity_probs"])
+                             for x, y in zip(el, elp) if y == max(elp)]
+            data["is_in_mention"].append(is_in_mention)
+            data["bioes"].append(convert_is_in_mention_to_bioes(is_in_mention))
+
+        maxlen = max([len(x) for x in data["tokens"]])
+        token_ids = torch.LongTensor([sample + [0] * (maxlen - len(sample)) for sample in data["tokens"]])
+        eval_mask = torch.LongTensor([sample + [0] * (maxlen - len(sample)) for sample in data["eval_mask"]])
+        is_in_mention = torch.LongTensor([sample + [0] * (maxlen - len(sample)) for sample in data["is_in_mention"]])
+        bioes = torch.LongTensor([sample + [2] * (maxlen - len(sample)) for sample in data["bioes"]])
+        if get_labels_with_high_model_score:
+            labels_with_high_model_score = get_labels_with_high_model_score(token_ids)
+        else:
+            labels_with_high_model_score = None
+        subword_mentions = create_output_with_negative_examples(
+            data["mentions"], data["mention_entity_probs"], token_ids.size(0), token_ids.size(1),
+            len(dl_sa.mentions_vocab), label_size, labels_with_high_model_score)
+        inputs = BatchEncoding({
+            'token_ids': token_ids,
+            'eval_mask': eval_mask,
+            'raw_mentions': data["mentions"],
+            'is_in_mention': is_in_mention,
+            "bioes": bioes
+        })
+        return inputs, subword_mentions
+    if not load_distributed or rank == 0:
+        print(f"Done initializing the {dataset_name.upper()}/{split} dataset ...")
+    wikipedia_dataset = ENWIKI20230827
+    wikipedia_dataset_config = ENWIKI20230827Config
+    retokenized_wikipedia_dataset = ENWIKI20230827V2
+    retokenized_wikipedia_dataset_config = ENWIKI20230827V2Config
+    aida_dataset = AIDA20230827
+    aida_dataset_config = AIDA20230827Config
+    dset_class = (retokenized_wikipedia_dataset if use_retokenized_wikipedia_data else wikipedia_dataset) \
+        if dataset_name == "enwiki" else aida_dataset
+    d_size = (retokenized_wikipedia_dataset_config.NUM_LINES[split] if use_retokenized_wikipedia_data else
+              wikipedia_dataset_config.NUM_LINES[split]) \
+        if dataset_name == "enwiki" else aida_dataset_config.NUM_LINES[split]
+    dataset_ = DistributableDataset(dset_class(split=split, root=get_checkpoints_dir()), d_size, world_size, rank) \
+        if load_distributed else dset_class(split=split, root=get_checkpoints_dir())
+    return DataLoader(dataset_, batch_size=batch_size, collate_fn=collate_batch,
+                      sampler=DistributedSampler(dataset_, num_replicas=world_size, rank=rank)) \
+        if load_distributed and split == "train" else DataLoader(dset_class(split=split, root=get_checkpoints_dir()),
+                                                                 batch_size=batch_size,
+                                                                 collate_fn=collate_batch)
+
+
+def create_output_with_negative_examples(batch_entity_ids, batch_entity_probs, batch_size, maxlen, label_vocab_size,
+                                         label_size, labels_with_high_model_score=None):
+    all_entity_ids = OrderedDict()
+    for batch_offset, (batch_item_token_item_entity_ids, batch_item_token_entity_probs) in enumerate(
+            zip(batch_entity_ids, batch_entity_probs)
+    ):
+        for tok_id, (token_entity_ids, token_entity_probs) in enumerate(
+                zip(batch_item_token_item_entity_ids, batch_item_token_entity_probs)
+        ):
+            for eid in token_entity_ids:
+                if eid not in all_entity_ids:
+                    all_entity_ids[eid] = len(all_entity_ids)
+    # #####################################################
+    shared_label_ids = list(all_entity_ids.keys())
+
+    if len(shared_label_ids) < label_size and labels_with_high_model_score is not None:
+        negative_examples = set(labels_with_high_model_score)
+        negative_examples.difference_update(shared_label_ids)
+        shared_label_ids += list(negative_examples)
+
+    if len(shared_label_ids) < label_size:
+        negative_samples = set(numpy.random.choice(label_vocab_size, label_size, replace=False))
+        negative_samples.difference_update(shared_label_ids)
+        shared_label_ids += list(negative_samples)
+
+    shared_label_ids = shared_label_ids[: label_size]
+
+    all_batch_entity_ids, batch_shared_label_ids = all_entity_ids, shared_label_ids
+    if label_size > 0:
+        label_probs = torch.zeros(batch_size, maxlen, len(batch_shared_label_ids))
+    else:
+        label_probs = torch.zeros(batch_size, maxlen, label_vocab_size)
+    # loop through the batch x tokens x (label_ids, label_probs)
+    for batch_offset, (batch_item_token_item_entity_ids, batch_item_token_entity_probs) in enumerate(
+            zip(batch_entity_ids, batch_entity_probs)
+    ):
+        # loop through tokens x (label_ids, label_probs)
+        for tok_id, (token_entity_ids, token_entity_probs) in enumerate(
+                zip(batch_item_token_item_entity_ids, batch_item_token_entity_probs)):
+            if label_size is None:
+                label_probs[batch_offset][tok_id][torch.LongTensor(token_entity_ids)] = torch.Tensor(
+                    batch_item_token_item_entity_ids)
+            else:
+                label_probs[batch_offset][tok_id][
+                    torch.LongTensor(list(map(all_batch_entity_ids.__getitem__, token_entity_ids)))
+                ] = torch.Tensor(token_entity_probs)
+
+    label_ids = torch.LongTensor(batch_shared_label_ids)
+    return BatchEncoding({
+        "ids": label_ids,  # of size label_size
+        "probs": label_probs,  # of size input_batch_size x input_max_len x label_size
+        "dictionary": {v: k for k, v in all_batch_entity_ids.items()}  # contains all original ids for mentions in batch
+    })
+
+
+def _make_vocab_file():
+    wiki_vocab = set()
+    vocab_file = open("enwiki_20230827.txt", "w")
+    for spl in ['train', 'valid', 'test']:
+        for el in tqdm(ENWIKI20230827(split=spl, root=get_checkpoints_dir())):
+            for x in el['mentions']:
+                for y in x:
+                    if y not in wiki_vocab:
+                        vocab_file.write(f"{y}\n")
+                    wiki_vocab.add(y)
+    vocab_file.close()

decao_eval.py

@@ -0,0 +1,1391 @@
+"""
+This file serves as a standalone evaluation provider for evaluating the predictions of a entity linking system.
+ The content of this module are taken from https://github.com/nicola-decao/efficient-autoregressive-EL and the necessary
+  boilerplate code is copied along with the metric classes to help the code act as standalone.
+
+To perform evaluation, import the following classes (or any subset of the evaluation metrics that you need):
+            MicroF1, MicroPrecision, MicroRecall, MacroRecall, MacroPrecision, MacroF1
+Collect the el_model predictions in the format of {(start_index, end_index, annotation string)} for document d.
+Collect the gold dataset annotations in the format of {(start_index, end_index, annotation string)} for document d.
+Call the metric instances for the two mentioned sets p and g:
+    micro_f1(p, g)
+    micro_prec(p, g)
+    micro_rec(p, g)
+    macro_f1(p, g)
+    macro_prec(p, g)
+    macro_rec(p, g)
+
+Once you are done with all the documents and all predictions are added, you may access the evaluation results using:
+    {'macro_f1': macro_f1.compute(),
+     'macro_prec': macro_prec.compute(),
+     'macro_rec': macro_rec.compute(),
+     'micro_f1': micro_f1.compute(),
+     'micro_prec': micro_prec.compute(),
+     'micro_rec': micro_rec.compute()}
+"""
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Hashable, Iterable, Generator, Sequence, Tuple, Union, List, Mapping, Callable, Optional
+import operator as op
+import functools
+import torch
+import torch.nn as nn
+from torch import Tensor
+import torch.nn.functional as F
+from contextlib import contextmanager
+import inspect
+from collections import OrderedDict
+from copy import deepcopy
+from importlib import import_module
+from importlib.util import find_spec
+
+from packaging.version import Version
+from pkg_resources import DistributionNotFound, get_distribution
+
+
+def dim_zero_sum(x: Tensor) -> Tensor:
+    """summation along the zero dimension."""
+    return torch.sum(x, dim=0)
+
+
+def dim_zero_mean(x: Tensor) -> Tensor:
+    """average along the zero dimension."""
+    return torch.mean(x, dim=0)
+
+
+def dim_zero_max(x: Tensor) -> Tensor:
+    """max along the zero dimension."""
+    return torch.max(x, dim=0).values
+
+
+def dim_zero_min(x: Tensor) -> Tensor:
+    """min along the zero dimension."""
+    return torch.min(x, dim=0).values
+
+
+def dim_zero_cat(x: Union[Tensor, List[Tensor]]) -> Tensor:
+    """concatenation along the zero dimension."""
+    x = x if isinstance(x, (list, tuple)) else [x]
+    x = [y.unsqueeze(0) if y.numel() == 1 and y.ndim == 0 else y for y in x]
+    if not x:  # empty list
+        raise ValueError("No samples to concatenate")
+    return torch.cat(x, dim=0)
+
+
+def _module_available(module_path: str) -> bool:
+    try:
+        return find_spec(module_path) is not None
+    except AttributeError:
+        # Python 3.6
+        return False
+    except ModuleNotFoundError:
+        # Python 3.7+
+        return False
+
+
+def _compare_version(package: str, op: Callable, version: str) -> Optional[bool]:
+    if not _module_available(package):
+        return None
+    try:
+        pkg = import_module(package)
+        pkg_version = pkg.__version__  # type: ignore
+    except (ModuleNotFoundError, DistributionNotFound):
+        return None
+    except ImportError:
+        # catches cyclic imports - the case with integrated libs
+        # see: https://stackoverflow.com/a/32965521
+        pkg_version = get_distribution(package).version
+    try:
+        pkg_version = Version(pkg_version)
+    except TypeError:
+        # this is mock by sphinx, so it shall return True ro generate all summaries
+        return True
+    return op(pkg_version, Version(version))
+
+
+class TorchMetricsUserError(Exception):
+    """Error used to inform users of a wrong combinison of Metric API calls."""
+
+
+def _simple_gather_all_tensors(result: Tensor, group: Any, world_size: int) -> List[Tensor]:
+    gathered_result = [torch.zeros_like(result) for _ in range(world_size)]
+    torch.distributed.all_gather(gathered_result, result, group)
+    return gathered_result
+
+
+def gather_all_tensors(result: Tensor, group: Optional[Any] = None) -> List[Tensor]:
+    """Function to gather all tensors from several ddp processes onto a list that is broadcasted to all processes.
+    Works on tensors that have the same number of dimensions, but where each dimension may differ. In this case
+    tensors are padded, gathered and then trimmed to secure equal workload for all processes.
+
+    Args:
+        result: the value to sync
+        group: the process group to gather results from. Defaults to all processes (world)
+
+    Return:
+        gathered_result: list with size equal to the process group where
+            gathered_result[i] corresponds to result tensor from process i
+    """
+    if group is None:
+        group = torch.distributed.group.WORLD
+
+    # convert tensors to contiguous format
+    result = result.contiguous()
+
+    world_size = torch.distributed.get_world_size(group)
+    torch.distributed.barrier(group=group)
+
+    # if the tensor is scalar, things are easy
+    if result.ndim == 0:
+        return _simple_gather_all_tensors(result, group, world_size)
+
+    # 1. Gather sizes of all tensors
+    local_size = torch.tensor(result.shape, device=result.device)
+    local_sizes = [torch.zeros_like(local_size) for _ in range(world_size)]
+    torch.distributed.all_gather(local_sizes, local_size, group=group)
+    max_size = torch.stack(local_sizes).max(dim=0).values
+    all_sizes_equal = all(all(ls == max_size) for ls in local_sizes)
+
+    # 2. If shapes are all the same, then do a simple gather:
+    if all_sizes_equal:
+        return _simple_gather_all_tensors(result, group, world_size)
+
+    # 3. If not, we need to pad each local tensor to maximum size, gather and then truncate
+    pad_dims = []
+    pad_by = (max_size - local_size).detach().cpu()
+    for val in reversed(pad_by):
+        pad_dims.append(0)
+        pad_dims.append(val.item())
+    result_padded = F.pad(result, pad_dims)
+    gathered_result = [torch.zeros_like(result_padded) for _ in range(world_size)]
+    torch.distributed.all_gather(gathered_result, result_padded, group)
+    for idx, item_size in enumerate(local_sizes):
+        slice_param = [slice(dim_size) for dim_size in item_size]
+        gathered_result[idx] = gathered_result[idx][slice_param]
+    return gathered_result
+
+
+def apply_to_collection(
+        data: Any,
+        dtype: Union[type, tuple],
+        function: Callable,
+        *args: Any,
+        wrong_dtype: Optional[Union[type, tuple]] = None,
+        **kwargs: Any,
+) -> Any:
+    """Recursively applies a function to all elements of a certain dtype.
+
+    Args:
+        data: the collection to apply the function to
+        dtype: the given function will be applied to all elements of this dtype
+        function: the function to apply
+        *args: positional arguments (will be forwarded to calls of ``function``)
+        wrong_dtype: the given function won't be applied if this type is specified and the given collections is of
+            the :attr:`wrong_type` even if it is of type :attr`dtype`
+        **kwargs: keyword arguments (will be forwarded to calls of ``function``)
+
+    Returns:
+        the resulting collection
+
+    Example:
+        >>> apply_to_collection(torch.tensor([8, 0, 2, 6, 7]), dtype=Tensor, function=lambda x: x ** 2)
+        tensor([64,  0,  4, 36, 49])
+        >>> apply_to_collection([8, 0, 2, 6, 7], dtype=int, function=lambda x: x ** 2)
+        [64, 0, 4, 36, 49]
+        >>> apply_to_collection(dict(abc=123), dtype=int, function=lambda x: x ** 2)
+        {'abc': 15129}
+    """
+    elem_type = type(data)
+
+    # Breaking condition
+    if isinstance(data, dtype) and (wrong_dtype is None or not isinstance(data, wrong_dtype)):
+        return function(data, *args, **kwargs)
+
+    # Recursively apply to collection items
+    if isinstance(data, Mapping):
+        return elem_type({k: apply_to_collection(v, dtype, function, *args, **kwargs) for k, v in data.items()})
+
+    if isinstance(data, tuple) and hasattr(data, "_fields"):  # named tuple
+        return elem_type(*(apply_to_collection(d, dtype, function, *args, **kwargs) for d in data))
+
+    if isinstance(data, Sequence) and not isinstance(data, str):
+        return elem_type([apply_to_collection(d, dtype, function, *args, **kwargs) for d in data])
+
+    # data is neither of dtype, nor a collection
+    return data
+
+
+def _flatten(x: Sequence) -> list:
+    return [item for sublist in x for item in sublist]
+
+
+def jit_distributed_available() -> bool:
+    return torch.distributed.is_available() and torch.distributed.is_initialized()
+
+
+class _Metric(nn.Module, ABC):
+    __jit_ignored_attributes__ = ["device"]
+    __jit_unused_properties__ = ["is_differentiable"]
+    is_differentiable: Optional[bool] = None
+    higher_is_better: Optional[bool] = None
+
+    def __init__(
+            self,
+            compute_on_step: bool = True,
+            dist_sync_on_step: bool = False,
+            process_group: Optional[Any] = None,
+            dist_sync_fn: Callable = None,
+    ) -> None:
+        super().__init__()
+
+        # see (https://github.com/pytorch/pytorch/blob/3e6bb5233f9ca2c5aa55d9cda22a7ee85439aa6e/
+        # torch/nn/modules/module.py#L227)
+        torch._C._log_api_usage_once(f"torchmetrics.metric.{self.__class__.__name__}")
+
+        self._LIGHTNING_GREATER_EQUAL_1_3 = _compare_version("pytorch_lightning", op.ge, "1.3.0")
+        self._device = torch.device("cpu")
+
+        self.dist_sync_on_step = dist_sync_on_step
+        self.compute_on_step = compute_on_step
+        self.process_group = process_group
+        self.dist_sync_fn = dist_sync_fn
+        self._to_sync = True
+        self._should_unsync = True
+
+        self._update_signature = inspect.signature(self.update)
+        self.update: Callable = self._wrap_update(self.update)  # type: ignore
+        self.compute: Callable = self._wrap_compute(self.compute)  # type: ignore
+        self._computed = None
+        self._forward_cache = None
+        self._update_called = False
+
+        # initialize state
+        self._defaults: Dict[str, Union[List, Tensor]] = {}
+        self._persistent: Dict[str, bool] = {}
+        self._reductions: Dict[str, Union[str, Callable[[Union[List[Tensor], Tensor]], Tensor], None]] = {}
+
+        # state management
+        self._is_synced = False
+        self._cache: Optional[Dict[str, Union[List[Tensor], Tensor]]] = None
+
+    def add_state(
+            self,
+            name: str,
+            default: Union[list, Tensor],
+            dist_reduce_fx: Optional[Union[str, Callable]] = None,
+            persistent: bool = False,
+    ) -> None:
+        if not isinstance(default, (Tensor, list)) or (isinstance(default, list) and default):
+            raise ValueError("state variable must be a tensor or any empty list (where you can append tensors)")
+
+        if dist_reduce_fx == "sum":
+            dist_reduce_fx = dim_zero_sum
+        elif dist_reduce_fx == "mean":
+            dist_reduce_fx = dim_zero_mean
+        elif dist_reduce_fx == "max":
+            dist_reduce_fx = dim_zero_max
+        elif dist_reduce_fx == "min":
+            dist_reduce_fx = dim_zero_min
+        elif dist_reduce_fx == "cat":
+            dist_reduce_fx = dim_zero_cat
+        elif dist_reduce_fx is not None and not callable(dist_reduce_fx):
+            raise ValueError("`dist_reduce_fx` must be callable or one of ['mean', 'sum', 'cat', None]")
+
+        if isinstance(default, Tensor):
+            default = default.contiguous()
+
+        setattr(self, name, default)
+
+        self._defaults[name] = deepcopy(default)
+        self._persistent[name] = persistent
+        self._reductions[name] = dist_reduce_fx
+
+    @torch.jit.unused
+    def forward(self, *args: Any, **kwargs: Any) -> Any:
+        """Automatically calls ``update()``.
+
+        Returns the metric value over inputs if ``compute_on_step`` is True.
+        """
+        # add current step
+        if self._is_synced:
+            raise TorchMetricsUserError(
+                "The Metric shouldn't be synced when performing ``update``. "
+                "HINT: Did you forget to call ``unsync`` ?."
+            )
+
+        with torch.no_grad():
+            self.update(*args, **kwargs)
+
+        if self.compute_on_step:
+            self._to_sync = self.dist_sync_on_step
+            # skip restore cache operation from compute as cache is stored below.
+            self._should_unsync = False
+
+            # save context before switch
+            cache = {attr: getattr(self, attr) for attr in self._defaults}
+
+            # call reset, update, compute, on single batch
+            self.reset()
+            self.update(*args, **kwargs)
+            self._forward_cache = self.compute()
+
+            # restore context
+            for attr, val in cache.items():
+                setattr(self, attr, val)
+            self._is_synced = False
+
+            self._should_unsync = True
+            self._to_sync = True
+            self._computed = None
+
+            return self._forward_cache
+
+    def _sync_dist(self, dist_sync_fn: Callable = gather_all_tensors, process_group: Optional[Any] = None) -> None:
+        input_dict = {attr: getattr(self, attr) for attr in self._reductions}
+
+        for attr, reduction_fn in self._reductions.items():
+            # pre-concatenate metric states that are lists to reduce number of all_gather operations
+            if reduction_fn == dim_zero_cat and isinstance(input_dict[attr], list) and len(input_dict[attr]) > 1:
+                input_dict[attr] = [dim_zero_cat(input_dict[attr])]
+
+        output_dict = apply_to_collection(
+            input_dict,
+            Tensor,
+            dist_sync_fn,
+            group=process_group or self.process_group,
+        )
+
+        for attr, reduction_fn in self._reductions.items():
+            # pre-processing ops (stack or flatten for inputs)
+            if isinstance(output_dict[attr][0], Tensor):
+                output_dict[attr] = torch.stack(output_dict[attr])
+            elif isinstance(output_dict[attr][0], list):
+                output_dict[attr] = _flatten(output_dict[attr])
+
+            if not (callable(reduction_fn) or reduction_fn is None):
+                raise TypeError("reduction_fn must be callable or None")
+            reduced = reduction_fn(output_dict[attr]) if reduction_fn is not None else output_dict[attr]
+            setattr(self, attr, reduced)
+
+    def _wrap_update(self, update: Callable) -> Callable:
+        @functools.wraps(update)
+        def wrapped_func(*args: Any, **kwargs: Any) -> Optional[Any]:
+            self._computed = None
+            self._update_called = True
+            return update(*args, **kwargs)
+
+        return wrapped_func
+
+    def sync(
+            self,
+            dist_sync_fn: Optional[Callable] = None,
+            process_group: Optional[Any] = None,
+            should_sync: bool = True,
+            distributed_available: Optional[Callable] = jit_distributed_available,
+    ) -> None:
+        """Sync function for manually controlling when metrics states should be synced across processes.
+
+        Args:
+            dist_sync_fn: Function to be used to perform states synchronization
+            process_group:
+                Specify the process group on which synchronization is called.
+                default: None (which selects the entire world)
+            should_sync: Whether to apply to state synchronization. This will have an impact
+                only when running in a distributed setting.
+            distributed_available: Function to determine if we are running inside a distributed setting
+        """
+        if self._is_synced and should_sync:
+            raise TorchMetricsUserError("The Metric has already been synced.")
+
+        is_distributed = distributed_available() if callable(distributed_available) else None
+
+        if not should_sync or not is_distributed:
+            return
+
+        if dist_sync_fn is None:
+            dist_sync_fn = gather_all_tensors
+
+        # cache prior to syncing
+        self._cache = {attr: getattr(self, attr) for attr in self._defaults}
+
+        # sync
+        self._sync_dist(dist_sync_fn, process_group=process_group)
+        self._is_synced = True
+
+    def unsync(self, should_unsync: bool = True) -> None:
+        """Unsync function for manually controlling when metrics states should be reverted back to their local
+        states.
+
+        Args:
+            should_unsync: Whether to perform unsync
+        """
+        if not should_unsync:
+            return
+
+        if not self._is_synced:
+            raise TorchMetricsUserError("The Metric has already been un-synced.")
+
+        if self._cache is None:
+            raise TorchMetricsUserError("The internal cache should exist to unsync the Metric.")
+
+        # if we synced, restore to cache so that we can continue to accumulate un-synced state
+        for attr, val in self._cache.items():
+            setattr(self, attr, val)
+        self._is_synced = False
+        self._cache = None
+
+    @contextmanager
+    def sync_context(
+            self,
+            dist_sync_fn: Optional[Callable] = None,
+            process_group: Optional[Any] = None,
+            should_sync: bool = True,
+            should_unsync: bool = True,
+            distributed_available: Optional[Callable] = jit_distributed_available,
+    ) -> Generator:
+        """Context manager to synchronize the states between processes when running in a distributed setting and
+        restore the local cache states after yielding.
+
+        Args:
+            dist_sync_fn: Function to be used to perform states synchronization
+            process_group:
+                Specify the process group on which synchronization is called.
+                default: None (which selects the entire world)
+            should_sync: Whether to apply to state synchronization. This will have an impact
+                only when running in a distributed setting.
+            should_unsync: Whether to restore the cache state so that the metrics can
+                continue to be accumulated.
+            distributed_available: Function to determine if we are running inside a distributed setting
+        """
+        self.sync(
+            dist_sync_fn=dist_sync_fn,
+            process_group=process_group,
+            should_sync=should_sync,
+            distributed_available=distributed_available,
+        )
+
+        yield
+
+        self.unsync(should_unsync=self._is_synced and should_unsync)
+
+    def _wrap_compute(self, compute: Callable) -> Callable:
+        @functools.wraps(compute)
+        def wrapped_func(*args: Any, **kwargs: Any) -> Any:
+            # return cached value
+            if self._computed is not None:
+                return self._computed
+
+            # compute relies on the sync context manager to gather the states across processes and apply reduction
+            # if synchronization happened, the current rank accumulated states will be restored to keep
+            # accumulation going if ``should_unsync=True``,
+            with self.sync_context(
+                    dist_sync_fn=self.dist_sync_fn, should_sync=self._to_sync, should_unsync=self._should_unsync
+            ):
+                self._computed = compute(*args, **kwargs)
+
+            return self._computed
+
+        return wrapped_func
+
+    @abstractmethod
+    def update(self, *_: Any, **__: Any) -> None:
+        """Override this method to update the state variables of your metric class."""
+
+    @abstractmethod
+    def compute(self) -> Any:
+        """Override this method to compute the final metric value from state variables synchronized across the
+        distributed backend."""
+
+    def reset(self) -> None:
+        """This method automatically resets the metric state variables to their default value."""
+        self._update_called = False
+        self._forward_cache = None
+        # lower lightning versions requires this implicitly to log metric objects correctly in self.log
+        self._computed = None
+
+        for attr, default in self._defaults.items():
+            current_val = getattr(self, attr)
+            if isinstance(default, Tensor):
+                setattr(self, attr, default.detach().clone().to(current_val.device))
+            else:
+                setattr(self, attr, [])
+
+        # reset internal states
+        self._cache = None
+        self._is_synced = False
+
+    def clone(self) -> "_Metric":
+        """Make a copy of the metric."""
+        return deepcopy(self)
+
+    def __getstate__(self) -> Dict[str, Any]:
+        # ignore update and compute functions for pickling
+        return {k: v for k, v in self.__dict__.items() if k not in ["update", "compute", "_update_signature"]}
+
+    def __setstate__(self, state: Dict[str, Any]) -> None:
+        # manually restore update and compute functions for pickling
+        self.__dict__.update(state)
+        self._update_signature = inspect.signature(self.update)
+        self.update: Callable = self._wrap_update(self.update)  # type: ignore
+        self.compute: Callable = self._wrap_compute(self.compute)  # type: ignore
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        if name in ("higher_is_better", "is_differentiable"):
+            raise RuntimeError(f"Can't change const `{name}`.")
+        super().__setattr__(name, value)
+
+    @property
+    def device(self) -> "torch.device":
+        """Return the device of the metric."""
+        return self._device
+
+    def type(self, dst_type: Union[str, torch.dtype]) -> "_Metric":
+        """Method override default and prevent dtype casting.
+
+        Please use `metric.set_dtype(dtype)` instead.
+        """
+        return self
+
+    def float(self) -> "_Metric":
+        """Method override default and prevent dtype casting.
+
+        Please use `metric.set_dtype(dtype)` instead.
+        """
+        return self
+
+    def double(self) -> "_Metric":
+        """Method override default and prevent dtype casting.
+
+        Please use `metric.set_dtype(dtype)` instead.
+        """
+        return self
+
+    def half(self) -> "_Metric":
+        """Method override default and prevent dtype casting.
+
+        Please use `metric.set_dtype(dtype)` instead.
+        """
+        return self
+
+    def set_dtype(self, dst_type: Union[str, torch.dtype]) -> None:
+        """Special version of `type` for transferring all metric states to specific dtype
+        Arguments:
+            dst_type (type or string): the desired type
+        """
+        return super().type(dst_type)
+
+    def _apply(self, fn: Callable) -> nn.Module:
+        """Overwrite _apply function such that we can also move metric states to the correct device when `.to`,
+        `.cuda`, etc methods are called."""
+        this = super()._apply(fn)
+        # Also apply fn to metric states and defaults
+        for key, value in this._defaults.items():
+            if isinstance(value, Tensor):
+                this._defaults[key] = fn(value)
+            elif isinstance(value, Sequence):
+                this._defaults[key] = [fn(v) for v in value]
+
+            current_val = getattr(this, key)
+            if isinstance(current_val, Tensor):
+                setattr(this, key, fn(current_val))
+            elif isinstance(current_val, Sequence):
+                setattr(this, key, [fn(cur_v) for cur_v in current_val])
+            else:
+                raise TypeError(
+                    "Expected metric state to be either a Tensor" f"or a list of Tensor, but encountered {current_val}"
+                )
+
+        # make sure to update the device attribute
+        # if the dummy tensor moves device by fn function we should also update the attribute
+        self._device = fn(torch.zeros(1, device=self.device)).device
+
+        # Additional apply to forward cache and computed attributes (may be nested)
+        if this._computed is not None:
+            this._computed = apply_to_collection(this._computed, Tensor, fn)
+        if this._forward_cache is not None:
+            this._forward_cache = apply_to_collection(this._forward_cache, Tensor, fn)
+
+        return this
+
+    def persistent(self, mode: bool = False) -> None:
+        """Method for post-init to change if metric states should be saved to its state_dict."""
+        for key in self._persistent:
+            self._persistent[key] = mode
+
+    def state_dict(
+            self,
+            destination: Dict[str, Any] = None,
+            prefix: str = "",
+            keep_vars: bool = False,
+    ) -> Optional[Dict[str, Any]]:
+        destination = super().state_dict(destination=destination, prefix=prefix, keep_vars=keep_vars)
+        # Register metric states to be part of the state_dict
+        for key in self._defaults:
+            if not self._persistent[key]:
+                continue
+            current_val = getattr(self, key)
+            if not keep_vars:
+                if isinstance(current_val, Tensor):
+                    current_val = current_val.detach()
+                elif isinstance(current_val, list):
+                    current_val = [cur_v.detach() if isinstance(cur_v, Tensor) else cur_v for cur_v in current_val]
+            destination[prefix + key] = deepcopy(current_val)  # type: ignore
+        return destination
+
+    def _load_from_state_dict(
+            self,
+            state_dict: dict,
+            prefix: str,
+            local_metadata: dict,
+            strict: bool,
+            missing_keys: List[str],
+            unexpected_keys: List[str],
+            error_msgs: List[str],
+    ) -> None:
+        """Loads metric states from state_dict."""
+
+        for key in self._defaults:
+            name = prefix + key
+            if name in state_dict:
+                setattr(self, key, state_dict.pop(name))
+        super()._load_from_state_dict(
+            state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs
+        )
+
+    def _filter_kwargs(self, **kwargs: Any) -> Dict[str, Any]:
+        """filter kwargs such that they match the update signature of the metric."""
+
+        # filter all parameters based on update signature except those of
+        # type VAR_POSITIONAL (*args) and VAR_KEYWORD (**kwargs)
+        _params = (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
+        _sign_params = self._update_signature.parameters
+        filtered_kwargs = {
+            k: v for k, v in kwargs.items() if (k in _sign_params.keys() and _sign_params[k].kind not in _params)
+        }
+
+        # if no kwargs filtered, return al kwargs as default
+        if not filtered_kwargs:
+            filtered_kwargs = kwargs
+        return filtered_kwargs
+
+    def __hash__(self) -> int:
+        # we need to add the id here, since PyTorch requires a module hash to be unique.
+        # Internally, PyTorch nn.Module relies on that for children discovery
+        # (see https://github.com/pytorch/pytorch/blob/v1.9.0/torch/nn/modules/module.py#L1544)
+        # For metrics that include tensors it is not a problem,
+        # since their hash is unique based on the memory location but we cannot rely on that for every metric.
+        hash_vals = [self.__class__.__name__, id(self)]
+
+        for key in self._defaults:
+            val = getattr(self, key)
+            # Special case: allow list values, so long
+            # as their elements are hashable
+            if hasattr(val, "__iter__") and not isinstance(val, Tensor):
+                hash_vals.extend(val)
+            else:
+                hash_vals.append(val)
+
+        return hash(tuple(hash_vals))
+
+    def __add__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.add, self, other)
+
+    def __and__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.bitwise_and, self, other)
+
+    # Fixme: this shall return bool instead of Metric
+    def __eq__(self, other: "Metric") -> "Metric":  # type: ignore
+        return CompositionalMetric(torch.eq, self, other)
+
+    def __floordiv__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.floor_divide, self, other)
+
+    def __ge__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.ge, self, other)
+
+    def __gt__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.gt, self, other)
+
+    def __le__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.le, self, other)
+
+    def __lt__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.lt, self, other)
+
+    def __matmul__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.matmul, self, other)
+
+    def __mod__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.fmod, self, other)
+
+    def __mul__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.mul, self, other)
+
+    # Fixme: this shall return bool instead of Metric
+    def __ne__(self, other: "Metric") -> "Metric":  # type: ignore
+        return CompositionalMetric(torch.ne, self, other)
+
+    def __or__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.bitwise_or, self, other)
+
+    def __pow__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.pow, self, other)
+
+    def __radd__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.add, other, self)
+
+    def __rand__(self, other: "Metric") -> "Metric":
+        # swap them since bitwise_and only supports that way and it's commutative
+        return CompositionalMetric(torch.bitwise_and, self, other)
+
+    def __rfloordiv__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.floor_divide, other, self)
+
+    def __rmatmul__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.matmul, other, self)
+
+    def __rmod__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.fmod, other, self)
+
+    def __rmul__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.mul, other, self)
+
+    def __ror__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.bitwise_or, other, self)
+
+    def __rpow__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.pow, other, self)
+
+    def __rsub__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.sub, other, self)
+
+    def __rtruediv__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.true_divide, other, self)
+
+    def __rxor__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.bitwise_xor, other, self)
+
+    def __sub__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.sub, self, other)
+
+    def __truediv__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.true_divide, self, other)
+
+    def __xor__(self, other: "Metric") -> "Metric":
+        return CompositionalMetric(torch.bitwise_xor, self, other)
+
+    def __abs__(self) -> "Metric":
+        return CompositionalMetric(torch.abs, self, None)
+
+    def __inv__(self) -> "Metric":
+        return CompositionalMetric(torch.bitwise_not, self, None)
+
+    def __invert__(self) -> "Metric":
+        return self.__inv__()
+
+    def __neg__(self) -> "Metric":
+        return CompositionalMetric(_neg, self, None)
+
+    def __pos__(self) -> "Metric":
+        return CompositionalMetric(torch.abs, self, None)
+
+    def __getitem__(self, idx: int) -> "Metric":
+        return CompositionalMetric(lambda x: x[idx], self, None)
+
+
+
+class CompositionalMetric(_Metric):
+    """Composition of two metrics with a specific operator which will be executed upon metrics compute."""
+
+    def __init__(
+            self,
+            operator: Callable,
+            metric_a: Union[_Metric, int, float, Tensor],
+            metric_b: Union[_Metric, int, float, Tensor, None],
+    ) -> None:
+        """
+        Args:
+            operator: the operator taking in one (if metric_b is None)
+                or two arguments. Will be applied to outputs of metric_a.compute()
+                and (optionally if metric_b is not None) metric_b.compute()
+            metric_a: first metric whose compute() result is the first argument of operator
+            metric_b: second metric whose compute() result is the second argument of operator.
+                For operators taking in only one input, this should be None
+        """
+        super().__init__()
+
+        self.op = operator
+
+        if isinstance(metric_a, Tensor):
+            self.register_buffer("metric_a", metric_a)
+        else:
+            self.metric_a = metric_a
+
+        if isinstance(metric_b, Tensor):
+            self.register_buffer("metric_b", metric_b)
+        else:
+            self.metric_b = metric_b
+
+    def _sync_dist(self, dist_sync_fn: Optional[Callable] = None, process_group: Optional[Any] = None) -> None:
+        # No syncing required here. syncing will be done in metric_a and metric_b
+        pass
+
+    def update(self, *args: Any, **kwargs: Any) -> None:
+        if isinstance(self.metric_a, Metric):
+            self.metric_a.update(*args, **self.metric_a._filter_kwargs(**kwargs))
+
+        if isinstance(self.metric_b, Metric):
+            self.metric_b.update(*args, **self.metric_b._filter_kwargs(**kwargs))
+
+    def compute(self) -> Any:
+
+        # also some parsing for kwargs?
+        if isinstance(self.metric_a, Metric):
+            val_a = self.metric_a.compute()
+        else:
+            val_a = self.metric_a
+
+        if isinstance(self.metric_b, Metric):
+            val_b = self.metric_b.compute()
+        else:
+            val_b = self.metric_b
+
+        if val_b is None:
+            return self.op(val_a)
+
+        return self.op(val_a, val_b)
+
+    def reset(self) -> None:
+        if isinstance(self.metric_a, Metric):
+            self.metric_a.reset()
+
+        if isinstance(self.metric_b, Metric):
+            self.metric_b.reset()
+
+    def persistent(self, mode: bool = False) -> None:
+        if isinstance(self.metric_a, Metric):
+            self.metric_a.persistent(mode=mode)
+        if isinstance(self.metric_b, Metric):
+            self.metric_b.persistent(mode=mode)
+
+    def __repr__(self) -> str:
+        _op_metrics = f"(\n  {self.op.__name__}(\n    {repr(self.metric_a)},\n    {repr(self.metric_b)}\n  )\n)"
+        repr_str = self.__class__.__name__ + _op_metrics
+
+        return repr_str
+
+
+class MetricCollection_(nn.ModuleDict):
+    def __init__(
+            self,
+            metrics: Union[_Metric, Sequence[_Metric], Dict[str, _Metric]],
+            *additional_metrics: _Metric,
+            prefix: Optional[str] = None,
+            postfix: Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        self.add_metrics(metrics, *additional_metrics)
+
+        self.prefix = self._check_arg(prefix, "prefix")
+        self.postfix = self._check_arg(postfix, "postfix")
+
+    @torch.jit.unused
+    def forward(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
+        """Iteratively call forward for each metric.
+
+        Positional arguments (args) will be passed to every metric in the collection, while keyword arguments (kwargs)
+        will be filtered based on the signature of the individual metric.
+        """
+        return {k: m(*args, **m._filter_kwargs(**kwargs)) for k, m in self.items()}
+
+    def update(self, *args: Any, **kwargs: Any) -> None:
+        """Iteratively call update for each metric.
+
+        Positional arguments (args) will be passed to every metric in the collection, while keyword arguments (kwargs)
+        will be filtered based on the signature of the individual metric.
+        """
+        for _, m in self.items(keep_base=True):
+            m_kwargs = m._filter_kwargs(**kwargs)
+            m.update(*args, **m_kwargs)
+
+    def compute(self) -> Dict[str, Any]:
+        return {k: m.compute() for k, m in self.items()}
+
+    def reset(self) -> None:
+        """Iteratively call reset for each metric."""
+        for _, m in self.items(keep_base=True):
+            m.reset()
+
+    def clone(self, prefix: Optional[str] = None, postfix: Optional[str] = None) -> "MetricCollection_":
+        """Make a copy of the metric collection
+        Args:
+            prefix: a string to append in front of the metric keys
+            postfix: a string to append after the keys of the output dict
+
+        """
+        mc = deepcopy(self)
+        if prefix:
+            mc.prefix = self._check_arg(prefix, "prefix")
+        if postfix:
+            mc.postfix = self._check_arg(postfix, "postfix")
+        return mc
+
+    def persistent(self, mode: bool = True) -> None:
+        """Method for post-init to change if metric states should be saved to its state_dict."""
+        for _, m in self.items(keep_base=True):
+            m.persistent(mode)
+
+    def add_metrics(
+            self, metrics: Union[_Metric, Sequence[_Metric], Dict[str, _Metric]], *additional_metrics: _Metric
+    ) -> None:
+        """Add new metrics to Metric Collection."""
+        if isinstance(metrics, Metric):
+            # set compatible with original type expectations
+            metrics = [metrics]
+        if isinstance(metrics, Sequence):
+            # prepare for optional additions
+            metrics = list(metrics)
+            remain: list = []
+            for m in additional_metrics:
+                (metrics if isinstance(m, Metric) else remain).append(m)
+
+        elif additional_metrics:
+            raise ValueError(
+                f"You have passes extra arguments {additional_metrics} which are not compatible"
+                f" with first passed dictionary {metrics} so they will be ignored."
+            )
+
+        if isinstance(metrics, dict):
+            # Check all values are metrics
+            # Make sure that metrics are added in deterministic order
+            for name in sorted(metrics.keys()):
+                metric = metrics[name]
+                if not isinstance(metric, Metric):
+                    raise ValueError(
+                        f"Value {metric} belonging to key {name} is not an instance of `pl.metrics.Metric`"
+                    )
+                self[name] = metric
+        elif isinstance(metrics, Sequence):
+            for metric in metrics:
+                if not isinstance(metric, Metric):
+                    raise ValueError(f"Input {metric} to `MetricCollection` is not a instance of `pl.metrics.Metric`")
+                name = metric.__class__.__name__
+                if name in self:
+                    raise ValueError(f"Encountered two metrics both named {name}")
+                self[name] = metric
+        else:
+            raise ValueError("Unknown input to MetricCollection.")
+
+    def _set_name(self, base: str) -> str:
+        name = base if self.prefix is None else self.prefix + base
+        name = name if self.postfix is None else name + self.postfix
+        return name
+
+    def _to_renamed_ordered_dict(self) -> OrderedDict:
+        od = OrderedDict()
+        for k, v in self._modules.items():
+            od[self._set_name(k)] = v
+        return od
+
+    def keys(self, keep_base: bool = False) -> Iterable[Hashable]:
+        r"""Return an iterable of the ModuleDict key.
+        Args:
+            keep_base: Whether to add prefix/postfix on the items collection.
+        """
+        if keep_base:
+            return self._modules.keys()
+        return self._to_renamed_ordered_dict().keys()
+
+    def items(self, keep_base: bool = False) -> Iterable[Tuple[str, nn.Module]]:
+        r"""Return an iterable of the ModuleDict key/value pairs.
+        Args:
+            keep_base: Whether to add prefix/postfix on the items collection.
+        """
+        if keep_base:
+            return self._modules.items()
+        return self._to_renamed_ordered_dict().items()
+
+    @staticmethod
+    def _check_arg(arg: Optional[str], name: str) -> Optional[str]:
+        if arg is None or isinstance(arg, str):
+            return arg
+        raise ValueError(f"Expected input `{name}` to be a string, but got {type(arg)}")
+
+    def __repr__(self) -> str:
+        repr_str = super().__repr__()[:-2]
+        if self.prefix:
+            repr_str += f",\n  prefix={self.prefix}{',' if self.postfix else ''}"
+        if self.postfix:
+            repr_str += f"{',' if not self.prefix else ''}\n  postfix={self.postfix}"
+        return repr_str + "\n)"
+
+
+class Metric(_Metric):
+    r"""
+    This implementation refers to :class:`~torchmetrics.Metric`.
+
+    .. warning:: This metric is deprecated, use ``torchmetrics.Metric``. Will be removed in v1.5.0.
+    """
+
+    def __init__(
+            self,
+            compute_on_step: bool = True,
+            dist_sync_on_step: bool = False,
+            process_group: Optional[Any] = None,
+            dist_sync_fn: Callable = None,
+    ):
+        super().__init__(
+            compute_on_step=compute_on_step,
+            dist_sync_on_step=dist_sync_on_step,
+            process_group=process_group,
+            dist_sync_fn=dist_sync_fn,
+        )
+
+    def __hash__(self):
+        return super().__hash__()
+
+    def __add__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.add, self, other)
+
+    def __and__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.bitwise_and, self, other)
+
+    def __eq__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.eq, self, other)
+
+    def __floordiv__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.floor_divide, self, other)
+
+    def __ge__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.ge, self, other)
+
+    def __gt__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.gt, self, other)
+
+    def __le__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.le, self, other)
+
+    def __lt__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.lt, self, other)
+
+    def __matmul__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.matmul, self, other)
+
+    def __mod__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.fmod, self, other)
+
+    def __mul__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.mul, self, other)
+
+    def __ne__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.ne, self, other)
+
+    def __or__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.bitwise_or, self, other)
+
+    def __pow__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.pow, self, other)
+
+    def __radd__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.add, other, self)
+
+    def __rand__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+
+        # swap them since bitwise_and only supports that way and it's commutative
+        return CompositionalMetric(torch.bitwise_and, self, other)
+
+    def __rfloordiv__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.floor_divide, other, self)
+
+    def __rmatmul__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.matmul, other, self)
+
+    def __rmod__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.fmod, other, self)
+
+    def __rmul__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.mul, other, self)
+
+    def __ror__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.bitwise_or, other, self)
+
+    def __rpow__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.pow, other, self)
+
+    def __rsub__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.sub, other, self)
+
+    def __rtruediv__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.true_divide, other, self)
+
+    def __rxor__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.bitwise_xor, other, self)
+
+    def __sub__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.sub, self, other)
+
+    def __truediv__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.true_divide, self, other)
+
+    def __xor__(self, other: Any):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.bitwise_xor, self, other)
+
+    def __abs__(self):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.abs, self, None)
+
+    def __inv__(self):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.bitwise_not, self, None)
+
+    def __invert__(self):
+        return self.__inv__()
+
+    def __neg__(self):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(_neg, self, None)
+
+    def __pos__(self):
+        from pytorch_lightning.metrics.compositional import CompositionalMetric
+        return CompositionalMetric(torch.abs, self, None)
+
+
+def _neg(tensor: torch.Tensor):
+    return -torch.abs(tensor)
+
+
+class MicroF1(Metric):
+    def __init__(self, dist_sync_on_step=False):
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.add_state("n", default=torch.tensor(0), dist_reduce_fx="sum")
+        self.add_state("prec_d", default=torch.tensor(0), dist_reduce_fx="sum")
+        self.add_state("rec_d", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def update(self, p, g):
+
+        self.n += len(g.intersection(p))
+        self.prec_d += len(p)
+        self.rec_d += len(g)
+
+    def compute(self):
+        p = self.n.float() / self.prec_d
+        r = self.n.float() / self.rec_d
+        return (2 * p * r / (p + r)) if (p + r) > 0 else (p + r)
+
+
+class MacroF1(Metric):
+    def __init__(self, dist_sync_on_step=False):
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.add_state("n", default=torch.tensor(0.0), dist_reduce_fx="sum")
+        self.add_state("d", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def update(self, p, g):
+
+        prec = len(g.intersection(p)) / len(p)
+        rec = len(g.intersection(p)) / len(g) if g else 0.0
+
+        self.n += (2 * prec * rec / (prec + rec)) if (prec + rec) > 0 else (prec + rec)
+        self.d += 1
+
+    def compute(self):
+        return (self.n / self.d) if self.d > 0 else self.d
+
+
+class MicroPrecision(Metric):
+    def __init__(self, dist_sync_on_step=False):
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.add_state("n", default=torch.tensor(0), dist_reduce_fx="sum")
+        self.add_state("d", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def update(self, p, g):
+        self.n += len(g.intersection(p))
+        self.d += len(p)
+
+    def compute(self):
+        return (self.n.float() / self.d) if self.d > 0 else self.d
+
+
+class MacroPrecision(Metric):
+    def __init__(self, dist_sync_on_step=False):
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.add_state("n", default=torch.tensor(0.0), dist_reduce_fx="sum")
+        self.add_state("d", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def update(self, p, g):
+        self.n += len(g.intersection(p)) / len(p)
+        self.d += 1
+
+    def compute(self):
+        return (self.n / self.d) if self.d > 0 else self.d
+
+
+class MicroRecall(Metric):
+    def __init__(self, dist_sync_on_step=False):
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.add_state("n", default=torch.tensor(0), dist_reduce_fx="sum")
+        self.add_state("d", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def update(self, p, g):
+        self.n += len(g.intersection(p))
+        self.d += len(g)
+
+    def compute(self):
+        return (self.n.float() / self.d) if self.d > 0 else self.d
+
+
+class MacroRecall(Metric):
+    def __init__(self, dist_sync_on_step=False):
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.add_state("n", default=torch.tensor(0.0), dist_reduce_fx="sum")
+        self.add_state("d", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def update(self, p, g):
+        self.n += len(g.intersection(p)) / len(g) if g else 0.0
+        self.d += 1
+
+    def compute(self):
+        return (self.n / self.d) if self.d > 0 else self.d
+
+# The following two classes are not inherited from https://github.com/nicola-decao/efficient-autoregressive-EL
+#  and are implemented in this project.
+
+
+class _EvaluationScores:
+    def __init__(self, is_micro):
+        self.is_micro = is_micro
+        if is_micro:
+            self.f1 = MicroF1()
+            self.p = MicroPrecision()
+            self.r = MicroRecall()
+        else:
+            self.f1 = MacroF1()
+            self.p = MacroPrecision()
+            self.r = MacroRecall()
+
+    def record_results(self, prediction, gold):
+        self.f1(prediction, gold)
+        self.p(prediction, gold)
+        self.r(prediction, gold)
+
+    def __str__(self):
+        im = "Micro" if self.is_micro else "Macro"
+        return f"\t{im} evaluation results: F1: {self.f1.compute() * 100:.3f}%\tP: {self.p.compute() * 100:.3f}%" \
+               f"\t R: {self.r.compute() * 100:.3f}%"
+
+
+class EntityEvaluationScores:
+    def __init__(self, dataset_name):
+        self.dataset_name = dataset_name
+        self.micro_mention_detection = _EvaluationScores(True)
+        self.macro_mention_detection = _EvaluationScores(False)
+        self.micro_entity_linking = _EvaluationScores(True)
+        self.macro_entity_linking = _EvaluationScores(False)
+
+    def record_mention_detection_results(self, prediction, gold):
+        self.micro_mention_detection.record_results(prediction, gold)
+        self.macro_mention_detection.record_results(prediction, gold)
+
+    def record_entity_linking_results(self, prediction, gold):
+        self.micro_entity_linking.record_results(prediction, gold)
+        self.macro_entity_linking.record_results(prediction, gold)
+
+    def __str__(self):
+        return f"Evaluated model for set: {self.dataset_name} (Entity Linking)\n" \
+               f"{str(self.macro_entity_linking)}\n" \
+               f"{str(self.micro_entity_linking)}\n" \
+               f"Evaluated model for set: {self.dataset_name} (Mention Detection)\n" \
+               f"{str(self.macro_mention_detection)}\n" \
+               f"{str(self.micro_mention_detection)}"
+
+
+class InOutMentionEvaluationResult:
+    def __init__(self, activation_threshold=0.5, vocab_index_of_o=-1):
+        self.activation_threshold = activation_threshold
+        self.vocab_index_of_o = vocab_index_of_o
+        self.total_predictions = 0.0
+        self.correct_predictions = 0.0
+        self.total_true_predictions = 0.0
+        self.correct_true_predictions = 0.0
+        self.total_false_predictions = 0.0
+        self.correct_false_predictions = 0.0
+
+    def _preprocess_logits(self, subword_logits):
+        if self.vocab_index_of_o > -1:
+            return (subword_logits.argmax(-1) != self.vocab_index_of_o).bool()
+        else:
+            return (subword_logits > self.activation_threshold).squeeze(-1)
+
+    def update_scores(self, inputs_eval_mask, s_mentions_is_in_mention, subword_logits):
+        self.total_predictions += inputs_eval_mask.sum().item()
+        for em, ac, pr in zip(inputs_eval_mask, s_mentions_is_in_mention.bool(),
+                              self._preprocess_logits(subword_logits)):
+            for m, a, p in zip(em, ac, pr):
+                if m:
+                    if a == p:
+                        self.correct_predictions += 1.0
+                    if a:
+                        self.total_true_predictions += 1.0
+                        if p:
+                            self.correct_true_predictions += 1.0
+                    else:
+                        self.total_false_predictions += 1.0
+                        if not p:
+                            self.correct_false_predictions += 1.0
+
+    @property
+    def overall_mention_detection_accuracy(self):
+        return self.correct_predictions * 100 / self.total_predictions if self.total_predictions > 0.0 else 0.0
+
+    @property
+    def in_mention_mention_detection_accuracy(self):
+        return self.correct_true_predictions * 100 / self.total_true_predictions \
+            if self.total_true_predictions > 0.0 else 0.0
+
+    @property
+    def out_of_mention_overall_mention_detection_accuracy(self):
+        return self.correct_false_predictions * 100 / self.total_false_predictions \
+            if self.total_false_predictions > 0.0 else 0.0
+
+    def __str__(self):
+        return f"Subword-level mention detection accuracy = {self.overall_mention_detection_accuracy:.3f}% " \
+               f"({int(self.correct_predictions)}/{int(self.total_predictions)})\n" \
+               f"\t    In-Mention Subword-level mention detection accuracy = " \
+               f"{self.in_mention_mention_detection_accuracy:.3f}% " \
+               f"({int(self.correct_true_predictions)}/{int(self.total_true_predictions)})\n" \
+               f"\tOut-of-Mention Subword-level mention detection accuracy = " \
+               f"{self.out_of_mention_overall_mention_detection_accuracy:.3f}% " \
+               f"({int(self.correct_false_predictions)}/{int(self.total_false_predictions)})"

model.py

@@ -0,0 +1,439 @@
+"""
+The implementation of the main annotator class from "SpEL: Structured Prediction for Entity Linking"
+"""
+import os
+import re
+import pickle
+import numpy
+from typing import List
+from glob import glob
+from itertools import chain
+
+from transformers import AutoModelForMaskedLM
+import torch
+import torch.nn as nn
+from torch import optim
+from tqdm import tqdm
+
+from utils import store_validation_data_wiki, chunk_annotate_and_merge_to_phrase, \
+    get_aida_set_phrase_splitted_documents, compare_gold_and_predicted_annotation_documents
+from decao_eval import EntityEvaluationScores, InOutMentionEvaluationResult
+from span_annotation import SubwordAnnotation
+from data_loader import BERT_MODEL_NAME, dl_sa, tokenizer
+from configuration import get_checkpoints_dir, get_aida_train_canonical_redirects, get_ood_canonical_redirects, \
+    get_logdir_dir, get_exec_run_file
+
+class SpELAnnotator:
+    def __init__(self):
+        super(SpELAnnotator, self).__init__()
+        self.checkpoints_root = get_checkpoints_dir()
+        self.logdir = get_logdir_dir()
+        self.exec_run_file = get_exec_run_file()
+
+        self.text_chunk_length = 254
+        self.text_chunk_overlap = 20
+
+        self.bert_lm = None
+        self.number_of_bert_layers = 0
+        self.bert_lm_h = 0
+        self.out = None
+        self.softmax = None
+
+    def init_model_from_scratch(self, base_model=BERT_MODEL_NAME, device="cpu"):
+        """
+        This is required to be called to load up the base model architecture before loading the fine-tuned checkpoint.
+        """
+        if base_model:
+            self.bert_lm = AutoModelForMaskedLM.from_pretrained(base_model, output_hidden_states=True,
+                                                                cache_dir=get_checkpoints_dir() / "hf").to(device)
+            self.disable_roberta_lm_head()
+            self.number_of_bert_layers = self.bert_lm.config.num_hidden_layers + 1
+            self.bert_lm_h = self.bert_lm.config.hidden_size
+            self.out = nn.Embedding(num_embeddings=len(dl_sa.mentions_vocab),
+                                    embedding_dim=self.bert_lm_h, sparse=True).to(device)
+            self.softmax = nn.Softmax(dim=-1)
+
+    def shrink_classification_head_to_aida(self, device):
+        """
+        This will be called in fine-tuning step 3 to shrink the classification head to in-domain data vocabulary.
+        """
+        aida_mentions_vocab, aida_mentions_itos = dl_sa.get_aida_vocab_and_itos()
+        if self.out_module.num_embeddings == len(aida_mentions_vocab):
+            return
+        current_state_dict = self.out_module.state_dict()
+        new_out = nn.Embedding(num_embeddings=len(aida_mentions_vocab),
+                               embedding_dim=self.bert_lm_h, sparse=True).to(device)
+        new_state_dict = new_out.state_dict()
+        for index_new in range(len(aida_mentions_itos)):
+            item_new = aida_mentions_itos[index_new]
+            assert item_new in dl_sa.mentions_vocab, \
+                "the aida fine-tuned mention vocab must be a subset of the original vocab"
+            index_current = dl_sa.mentions_vocab[item_new]
+            new_state_dict['weight'][index_new] = current_state_dict['weight'][index_current]
+        new_out.load_state_dict(new_state_dict, strict=False)
+        self.out = new_out.to(device)
+        dl_sa.shrink_vocab_to_aida()
+        model_params = sum(p.numel() for p in self.bert_lm.parameters())
+        out_params = sum(p.numel() for p in self.out.parameters())
+        print(f' * Shrank model to {model_params+out_params} number of parameters ({model_params} parameters '
+              f'for the encoder and {out_params} parameters for the classification head)!')
+
+    @property
+    def current_device(self):
+        return self.lm_module.device
+
+    @property
+    def lm_module(self):
+        return self.bert_lm.module if isinstance(self.bert_lm, nn.DataParallel) or \
+                                      isinstance(self.bert_lm, nn.parallel.DistributedDataParallel) else self.bert_lm
+
+    @property
+    def out_module(self):
+        return self.out.module if isinstance(self.out, nn.DataParallel) or \
+                                  isinstance(self.out, nn.parallel.DistributedDataParallel) else self.out
+
+    @staticmethod
+    def get_canonical_redirects(limit_to_conll=True):
+        return get_aida_train_canonical_redirects() if limit_to_conll else get_ood_canonical_redirects()
+
+    def create_optimizers(self, encoder_lr=5e-5, decoder_lr=0.1, exclude_parameter_names_regex=None):
+        if exclude_parameter_names_regex is not None:
+            bert_lm_parameters = list()
+            regex = re.compile(exclude_parameter_names_regex)
+            for n, p in list(self.lm_module.named_parameters()):
+                if not len(regex.findall(n)) > 0:
+                    bert_lm_parameters.append(p)
+        else:
+            bert_lm_parameters = list(self.lm_module.parameters())
+        bert_optim = optim.Adam(bert_lm_parameters, lr=encoder_lr)
+        if decoder_lr < 1e-323:
+            # IMPORTANT! This is a hack since if we don't consider an optimizer for the last layer(e.g. decoder_lr=0.0),
+            #  BCEWithLogitsLoss will become unstable and memory will explode.
+            decoder_lr = 1e-323
+        out_optim = optim.SparseAdam(self.out.parameters(), lr=decoder_lr)
+        return bert_optim, out_optim
+
+    @staticmethod
+    def create_warmup_scheduler(optimizer, warmup_steps):
+        """
+        Creates a scheduler which increases the :param optimizer: learning rate from 0 to the specified learning rate
+            in :param warmup_steps: number of batches.
+        You need to call scheduler.step() after optimizer.step() in your code for this scheduler to take effect
+        """
+        return optim.lr_scheduler.LambdaLR(
+            optimizer, lambda epoch: epoch / warmup_steps if epoch < warmup_steps else 1.0)
+
+    def get_highest_confidence_model_predictions(self, batch_token_ids, topk_per_token=20, topk_from_batch=8196):
+        """
+        This function will be used for hard negative mining. For a given input batch, it will return
+         the `topk_from_batch` mentions which have had model puzzled. In the process, to reduce the computational
+          complexity the model will first select `topk_per_token` number of candidates from the vocabulary, and then
+           applies the topk selection on it.
+        """
+        with torch.no_grad():
+            logits = self.get_model_raw_logits_inference(batch_token_ids)
+            # topk_logit_per_token, topk_eids_per_token = logits.topk(topk_per_token, sorted=False, dim=-1)
+            # This is a workaround to the torch.topk bug for large sized tensors
+            topk_logit_per_token, topk_eids_per_token = [], []
+            for batch_item in logits:
+                topk_probs, topk_ids = batch_item.topk(topk_per_token, sorted=False, dim=-1)
+                topk_logit_per_token.append(topk_probs)
+                topk_eids_per_token.append(topk_ids)
+            topk_logit_per_token = torch.stack(topk_logit_per_token, dim=0)
+            topk_eids_per_token = torch.stack(topk_eids_per_token, dim=0)
+            i = torch.cat(
+                [
+                    topk_eids_per_token.view(1, -1),
+                    torch.zeros(topk_eids_per_token.view(-1).size(), dtype=torch.long,
+                                device=topk_eids_per_token.device).view(1, -1),
+                ],
+                dim=0,
+            )
+            v = topk_logit_per_token.view(-1)
+            st = torch.sparse.FloatTensor(i, v)
+            stc = st.coalesce()
+            topk_indices = stc._values().sort(descending=True)[1][:topk_from_batch]
+            result = stc._indices()[0, topk_indices]
+
+            return result.cpu().tolist()
+            # ###########################################################################################
+
+    def annotate_subword_ids(self, subword_ids_list: List, k_for_top_k_to_keep: int, token_offsets=None) \
+            -> List[SubwordAnnotation]:
+        with torch.no_grad():
+            token_ids = torch.LongTensor(subword_ids_list)
+            raw_logits, hidden_states = self.get_model_raw_logits_inference(token_ids, return_hidden_states=True)
+            logits = self.get_model_logits_inference(raw_logits, hidden_states, k_for_top_k_to_keep, token_offsets)
+            return logits
+
+    def get_model_raw_logits_training(self, token_ids, label_ids, label_probs):
+        # label_probs is not used in this function but provided for the classes inheriting SpELAnnotator.
+        enc = self.bert_lm(token_ids).hidden_states[-1]
+        out = self.out(label_ids)
+        logits = enc.matmul(out.transpose(0, 1))
+        return logits
+
+    def get_model_logits_inference(self, raw_logits, hidden_states, k_for_top_k_to_keep, token_offsets=None) \
+            -> List[SubwordAnnotation]:
+        # hidden_states is not used in this function but provided for the classes inheriting SpELAnnotator.
+        logits = self.softmax(raw_logits)
+        # The following line could possibly cause errors in torch version 1.13.1
+        # see https://github.com/pytorch/pytorch/issues/95455 for more information
+        top_k_logits, top_k_indices = logits.topk(k_for_top_k_to_keep)
+        top_k_logits = top_k_logits.squeeze(0).cpu().tolist()
+        top_k_indices = top_k_indices.squeeze(0).cpu().tolist()
+        chunk = ["" for _ in top_k_logits] if token_offsets is None else token_offsets
+        return [SubwordAnnotation(p, i, x[0]) for p, i, x in zip(top_k_logits, top_k_indices, chunk)]
+
+    def get_model_raw_logits_inference(self, token_ids, return_hidden_states=False):
+        encs = self.lm_module(token_ids.to(self.current_device)).hidden_states
+        out = self.out_module.weight
+        logits = encs[-1].matmul(out.transpose(0, 1))
+        return (logits, encs) if return_hidden_states else logits
+
+    def evaluate(self, epoch, batch_size, label_size, best_f1, is_training=True, use_retokenized_wikipedia_data=False,
+                 potent_score_threshold=0.82):
+        self.bert_lm.eval()
+        self.out.eval()
+        vocab_pad_id = dl_sa.mentions_vocab['<pad>']
+
+        all_words, all_tags, all_y, all_y_hat, all_predicted, all_token_ids = [], [], [], [], [], []
+        subword_eval = InOutMentionEvaluationResult(vocab_index_of_o=dl_sa.mentions_vocab['|||O|||'])
+        dataset_name = store_validation_data_wiki(
+            self.checkpoints_root, batch_size, label_size, is_training=is_training,
+            use_retokenized_wikipedia_data=use_retokenized_wikipedia_data)
+        with torch.no_grad():
+            for d_file in tqdm(sorted(glob(os.path.join(self.checkpoints_root, dataset_name, "*")))):
+                batch_token_ids, label_ids, label_probs, eval_mask, label_id_to_entity_id_dict, \
+                    batch_entity_ids, is_in_mention, _ = pickle.load(open(d_file, "rb"))
+                logits = self.get_model_raw_logits_inference(batch_token_ids)
+                subword_eval.update_scores(eval_mask, is_in_mention, logits)
+                y_hat = logits.argmax(-1)
+
+                tags = list()
+                predtags = list()
+                y_resolved_list = list()
+                y_hat_resolved_list = list()
+                token_list = list()
+
+                for batch_id, seq in enumerate(label_probs.max(-1)[1]):
+                    for token_id, label_id in enumerate(seq[:-self.text_chunk_overlap]):
+                        if eval_mask[batch_id][token_id].item() == 0:
+                            y_resolved = vocab_pad_id
+                        else:
+                            y_resolved = label_ids[label_id].item()
+                        y_resolved_list.append(y_resolved)
+                        tags.append(dl_sa.mentions_itos[y_resolved])
+                        y_hat_resolved = y_hat[batch_id][token_id].item()
+                        y_hat_resolved_list.append(y_hat_resolved)
+                        predtags.append(dl_sa.mentions_itos[y_hat_resolved])
+                        token_list.append(batch_token_ids[batch_id][token_id].item())
+
+                all_y.append(y_resolved_list)
+                all_y_hat.append(y_hat_resolved_list)
+                all_tags.append(tags)
+                all_predicted.append(predtags)
+                all_words.append(tokenizer.convert_ids_to_tokens(token_list))
+                all_token_ids.append(token_list)
+                del batch_token_ids, label_ids, label_probs, eval_mask, \
+                    label_id_to_entity_id_dict, batch_entity_ids, logits, y_hat
+
+        y_true = numpy.array(list(chain(*all_y)))
+        y_pred = numpy.array(list(chain(*all_y_hat)))
+        all_token_ids = numpy.array(list(chain(*all_token_ids)))
+
+        num_proposed = len(y_pred[(1 < y_pred) & (all_token_ids > 0)])
+        num_correct = (((y_true == y_pred) & (1 < y_true) & (all_token_ids > 0))).astype(int).sum()
+        num_gold = len(y_true[(1 < y_true) & (all_token_ids > 0)])
+
+        precision = num_correct / num_proposed if num_proposed > 0.0 else 0.0
+        recall = num_correct / num_gold if num_gold > 0.0 else 0.0
+        f1 = 2.0 * precision * recall / (precision + recall) if precision + recall > 0.0 else 0.0
+        f05 = 1.5 * precision * recall / (precision + recall) if precision + recall > 0.0 else 0.0
+        if f1 > best_f1:
+            print("Saving the best checkpoint ...")
+            config = self.prepare_model_checkpoint(epoch)
+            fname = self.get_mode_checkpoint_name()
+            torch.save(config, f"{fname}.pt")
+            print(f"weights were saved to {fname}.pt")
+        if precision > potent_score_threshold and recall > potent_score_threshold and is_training:
+            print(f"Saving the potent checkpoint with both precision and recall above {potent_score_threshold} ...")
+            config = self.prepare_model_checkpoint(epoch)
+            try:
+                fname = self.get_mode_checkpoint_name()
+                torch.save(config, f"{fname}-potent.pt")
+                print(f"weights were saved to {fname}-potent.pt")
+            except NotImplementedError:
+                pass
+        self.bert_lm.train()
+        self.out.train()
+        with open(self.exec_run_file, "a+") as exec_file:
+            exec_file.write(f"{precision}, {recall}, {f1}, {f05}, {num_proposed}, {num_correct}, {num_gold}, "
+                            f"{epoch+1},,\n")
+        return precision, recall, f1, f05, num_proposed, num_correct, num_gold, subword_eval
+
+    def inference_evaluate(self, epoch, best_f1, dataset_name='testa'):
+        self.bert_lm.eval()
+        self.out.eval()
+        evaluation_results = EntityEvaluationScores(dataset_name)
+        gold_documents = get_aida_set_phrase_splitted_documents(dataset_name)
+        for gold_document in tqdm(gold_documents):
+            t_sentence = " ".join([x.word_string for x in gold_document])
+            predicted_document = chunk_annotate_and_merge_to_phrase(self, t_sentence, k_for_top_k_to_keep=1)
+            comparison_results = compare_gold_and_predicted_annotation_documents(gold_document, predicted_document)
+            g_md = set((e[1].begin_character, e[1].end_character)
+                       for e in comparison_results if e[0].resolved_annotation)
+            p_md = set((e[1].begin_character, e[1].end_character)
+                       for e in comparison_results if e[1].resolved_annotation)
+            g_el = set((e[1].begin_character, e[1].end_character, dl_sa.mentions_itos[e[0].resolved_annotation])
+                       for e in comparison_results if e[0].resolved_annotation)
+            p_el = set((e[1].begin_character, e[1].end_character, dl_sa.mentions_itos[e[1].resolved_annotation])
+                       for e in comparison_results if e[1].resolved_annotation)
+            if p_el:
+                evaluation_results.record_mention_detection_results(p_md, g_md)
+                evaluation_results.record_entity_linking_results(p_el, g_el)
+        if evaluation_results.micro_entity_linking.f1.compute() > best_f1:
+            print("Saving the best checkpoint ...")
+            config = self.prepare_model_checkpoint(epoch)
+            fname = self.get_mode_checkpoint_name()
+            torch.save(config, f"{fname}.pt")
+            print(f"weights were saved to {fname}.pt")
+        self.bert_lm.train()
+        self.out.train()
+        return evaluation_results
+
+    def prepare_model_checkpoint(self, epoch):
+        chk_point = {
+            "bert_lm": self.lm_module.state_dict(),
+            "number_of_bert_layers": self.number_of_bert_layers,
+            "bert_lm_h": self.bert_lm_h,
+            "out": self.out_module.state_dict(),
+            "epoch": epoch,
+        }
+        sub_model_specific_checkpoint_data = self.sub_model_specific_checkpoint_data()
+        for key in sub_model_specific_checkpoint_data:
+            assert key not in ["bert_lm", "number_of_bert_layers", "bert_lm_h", "out", "epoch"], \
+                f"{key} is already considered in prepare_model_checkpoint function"
+            chk_point[key] = sub_model_specific_checkpoint_data[key]
+        return chk_point
+
+    def disable_roberta_lm_head(self):
+        assert self.bert_lm is not None
+        self.bert_lm.lm_head.layer_norm.bias.requires_grad = False
+        self.bert_lm.lm_head.layer_norm.weight.requires_grad = False
+        self.bert_lm.lm_head.dense.bias.requires_grad = False
+        self.bert_lm.lm_head.dense.weight.requires_grad = False
+        self.bert_lm.lm_head.decoder.bias.requires_grad = False
+
+    def _load_from_checkpoint_object(self, checkpoint, device="cpu"):
+        torch.cuda.empty_cache()
+        self.bert_lm.load_state_dict(checkpoint["bert_lm"], strict=False)
+        self.bert_lm.to(device)
+        self.disable_roberta_lm_head()
+        self.out.load_state_dict(checkpoint["out"], strict=False)
+        self.out.to(device)
+        self.number_of_bert_layers = checkpoint["number_of_bert_layers"]
+        self.bert_lm_h = checkpoint["bert_lm_h"]
+        self.sub_model_specific_load_checkpoint_data(checkpoint)
+        self.bert_lm.eval()
+        self.out.eval()
+        model_params = sum(p.numel() for p in self.bert_lm.parameters())
+        out_params = sum(p.numel() for p in self.out.parameters())
+        print(f' * Loaded model with {model_params+out_params} number of parameters ({model_params} parameters '
+              f'for the encoder and {out_params} parameters for the classification head)!')
+
+    @staticmethod
+    def download_from_torch_hub(finetuned_after_step=1):
+        assert 4 >= finetuned_after_step >= 1
+        if finetuned_after_step == 4:
+            # This model is the same SpEL finetuned model after step 3 except that its classification layer projects to
+            # the entirety of the step-2 model rather than shrinking it in size
+            file_name = "spel-base-step-3-500K.pt"
+            # Downloads and returns the finetuned model checkpoint created on Oct-03-2023
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/8nw5fFXdz2yBP5z/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        elif finetuned_after_step == 3:
+            file_name = "spel-base-step-3.pt"
+            # Downloads and returns the finetuned model checkpoint created on Sep-26-2023 with P=92.06|R=91.93|F1=91.99
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/HpQ3PMm6A3y1NBl/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        elif finetuned_after_step == 2:
+            file_name = 'spel-base-step-2.pt'
+            # Downloads and returns the pretrained model checkpoint created on Sep-26-2023 with P=77.60|R=77.91|F1=77.75
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/Hf37vc1foluHPBh/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        else:
+            file_name = 'spel-base-step-1.pt'
+            # Downloads and returns the pretrained model checkpoint created on Sep-11-2023 with P=82.50|R=83.16|F1=82.83
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/9OAoAG5eYeREE9V/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        print(f" * Loaded pretrained model checkpoint: {file_name}")
+        return checkpoint
+
+    @staticmethod
+    def download_large_from_torch_hub(finetuned_after_step=1):
+        assert 4 >= finetuned_after_step >= 1
+        if finetuned_after_step == 4:
+            # This model is the same SpEL finetuned model after step 3 except that its classification layer projects to
+            # the entirety of the step-2 model rather than shrinking it in size
+            file_name = "spel-large-step-3-500K.pt"
+            # Downloads and returns the finetuned model checkpoint created on Oct-03-2023
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/BCvputD1ByAvILC/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        elif finetuned_after_step == 3:
+            file_name = "spel-large-step-3.pt"
+            # Downloads and returns the finetuned model checkpoint created on Oct-02-2023 with P=92.53|R=92.99|F1=93.76
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/kBBlYVM4Tr59P0q/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        elif finetuned_after_step == 2:
+            file_name = 'spel-large-step-2.pt'
+            # Downloads and returns the pretrained model checkpoint created on Oct-02-2023 with P=77.36|R=73.11|F1=75.18
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/rnDiuKns7gzADyb/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        else:
+            file_name = 'spel-large-step-1.pt'
+            # Downloads and returns the pretrained model checkpoint created on Sep-11-2023 with P=84.02|R=82.74|F1=83.37
+            checkpoint = torch.hub.load_state_dict_from_url('https://vault.sfu.ca/index.php/s/bTp6UN2xL7Yh52w/download',
+                                                            model_dir=str(get_checkpoints_dir()), map_location="cpu",
+                                                            file_name=file_name)
+        print(f" * Loaded pretrained model checkpoint: {file_name}")
+        return checkpoint
+
+
+    def load_checkpoint(self, checkpoint_name, device="cpu", rank=0, load_from_torch_hub=False, finetuned_after_step=1):
+        if load_from_torch_hub and BERT_MODEL_NAME == "roberta-large":
+            checkpoint = self.download_large_from_torch_hub(finetuned_after_step)
+            self._load_from_checkpoint_object(checkpoint, device)
+        elif load_from_torch_hub and BERT_MODEL_NAME == "roberta-base":
+            checkpoint = self.download_from_torch_hub(finetuned_after_step)
+            self._load_from_checkpoint_object(checkpoint, device)
+        else: # load from the local .checkpoints directory
+            if rank == 0:
+                print("Loading model checkpoint: {}".format(checkpoint_name))
+            fname = os.path.join(self.checkpoints_root, checkpoint_name)
+            checkpoint = torch.load(fname, map_location="cpu")
+            self._load_from_checkpoint_object(checkpoint, device)
+
+    # #############################FUNCTIONS THAT THE SUB-MODELS MUST REIMPLEMENT####################################
+    def sub_model_specific_checkpoint_data(self):
+        """
+        :return: a dictionary of key values containing everything that matters to the sub-model and is not already
+            considered in prepare_model_checkpoint.
+        """
+        return {}
+
+    def sub_model_specific_load_checkpoint_data(self, checkpoint):
+        return
+
+    def get_mode_checkpoint_name(self):
+        raise NotImplementedError
+
+    def annotate(self, nif_collection, **kwargs):
+        raise NotImplementedError

resources/vocab/aida.txt

@@ -0,0 +1,5598 @@
+1._FC_Köln
+1._FC_Tatran_Prešov
+1._HFC_Humenné
+10_Downing_Street
+14th_Dalai_Lama
+1936_Summer_Olympics
+1966_FIFA_World_Cup
+1972_Summer_Olympics
+1976_Winter_Olympics
+1990_FIFA_World_Cup
+1992_Summer_Olympics
+1994_Asian_Games
+1995_Rugby_World_Cup
+1996_AFC_Asian_Cup
+1996_Belgian_Grand_Prix
+1996_IAAF_Grand_Prix_Final
+1996_Summer_Olympics
+1996_Wimbledon_Championships
+1996–97_UEFA_Cup
+1997_Fed_Cup
+1998_FIFA_World_Cup
+2,000_Guineas_Stakes
+2000_Summer_Olympics
+7-Eleven_(cycling_team)
+A-Ram
+A.C._Cesena
+A.C._ChievoVerona
+A.C._Milan
+A.C._Monza_Brianza_1912
+A.C._Reggiana_1919
+A.F.C._Bournemouth
+A.S.G._Nocerina
+A.S._Andria_BAT
+A.S._Bari
+A.S._Lucchese_Libertas_1905
+A.S._Roma
+A._G._Edwards
+ABC_(newspaper)
+ABN_AMRO
+ACC_Limited
+ACES_Colombia
+ACF_Fiorentina
+ACF_Gloria_1922_Bistriţa
+AEK_Athens_F.C.
+AEK_Larnaca
+AES_Eletropaulo
+AEX_index
+AFC_Ajax
+AFC_Asian_Cup
+AFC_Progresul_Bucureşti
+AIDS
+AJ_Auxerre
+AK-47
+ALBA_Berlin
+AMR_Corporation
+AOL
+ASARCO
+ASEAN
+ASEC_Mimosas
+ASVEL_Lyon-Villeurbanne
+AS_Cannes
+AS_Monaco_FC
+AS_Nancy
+AT&T_Classic
+AZ_(football_club)
+A_Coruña
+Aamer_Sohail
+Aaron_Slight
+Aashish_Kapoor
+Abdalá_Bucaram
+Abdou_Diouf
+Abdul_Rashid_Dostum
+Abdullah_Ercan
+Abdullah_of_Saudi_Arabia
+Abel_Antón
+Abel_Balbo
+Aberdeen_F.C.
+Abidjan
+Abimael_Guzmán
+Abraham_Lincoln
+Abu_Dhabi
+Abu_Dhabi_(emirate)
+Abuja
+Abulhassan_Banisadr
+Academic_Staff_Union_of_Universities
+Academy_Award
+Academy_Award_for_Best_Actor
+Acatepec
+Action_Against_Hunger
+Ad-Diyar
+Adam_Fedoruk
+Adam_Hollioake
+Adam_Hunter_(golfer)
+Adelaide_Football_Club
+Adige
+Adilson_da_Silva
+Adnan_Al_Talyani
+Adolf_Hitler
+Adrian_Ilie
+Adrian_Knup
+Adrian_Năstase
+Adrian_Voinea
+Adriatic_Sea
+Aegon
+Afghanistan
+Africa
+Africa_Cup_of_Nations
+African_American
+African_National_Congress
+Afrikaans
+Afrikaner
+Agent_Orange
+Agenzia_Nazionale_Stampa_Associata
+Agnieszka_Kotlarska
+Agung_Setyabudi
+Ahmad_Shah_Massoud
+Ahmed_Barada
+Ahmed_Sékou_Touré
+Ahmed_Yassin
+Ahmedou_Ould-Abdallah
+Ahold
+Ai_Sugiyama
+Aidan_Quinn
+Aimé_Jacquet
+Air_France
+Airdrieonians_F.C.
+Ajaccio
+Ajay_Jadeja
+Akira_Ryō
+Akron,_Ohio
+Al-Aqsa_Mosque
+Al_Ain
+Al_Akhbar_(Lebanon)
+Al_Gore
+Al_Karak
+Al_Martin
+Al_Rai
+Al_Unser,_Jr.
+Alabama
+Alain_Caveglia
+Alain_Juppé
+Alama_Ieremia
+Alan_Ball,_Jr.
+Alan_Benes
+Alan_Budikusuma
+Alan_Greenspan
+Alan_Hunte
+Alan_Kelly,_Sr.
+Alan_Kernaghan
+Alan_McLoughlin
+Alan_Moore_(footballer)
+Alan_Mullally
+Alan_Shearer
+Alaska
+Alaska_Aces_(PBA)
+Albania
+Albania_national_football_team
+Albanian_Football_Association
+Albanian_language
+Albanians
+Albert_Belle
+Albert_Emon
+Albert_Ferrer
+Albert_II,_Prince_of_Monaco
+Albert_Schweitzer
+Alberta
+Alberto_Berasategui
+Alberto_García_Aspe
+Albuquerque,_New_Mexico
+Aldi
+Alec_Stewart
+Alejandro_Agustín_Lanusse
+Aleksander_Kwaśniewski
+Aleksandra_Olsza
+Alen_Bokšić
+Alessandra_Mussolini
+Alessandro_Del_Piero
+Alessandro_Lambruschini
+Alessandro_Melli
+Alessandro_Moscardi
+Alessandro_Troncon
+Alex_Arias
+Alex_Ferguson
+Alex_Fernandez_(baseball)
+Alex_Rodriguez
+Alex_Rădulescu
+Alex_Zanardi
+Alex_Čejka
+Alexander_Downer
+Alexander_Gontchenkov
+Alexander_III_of_Scotland
+Alexander_Lebed
+Alexander_Lukashenko
+Alexander_Popov_(swimmer)
+Alexander_Vladimirovich_Volkov
+Alexander_Zickler
+Alexandra_Fusai
+Alexandra_Meissnitzer
+Alexandre_Comisetti
+Alexei_Markov
+Alexia_Dechaume-Balleret
+Aleš_Valenta
+Alfred_Berkeley
+Alfred_Sant
+Algemeen_Nederlands_Persbureau
+Algeria
+Algiers
+Ali_Alatas
+Ali_Brown
+Ali_Shah
+Alicante
+Alicia_Machado
+Alija_Izetbegović
+Alina_Astafei
+Alistair_Campbell_(cricketer)
+Aliuska_López
+Alla_Dudayeva
+Allah
+Allan_Bateman
+Allan_Donald
+Allan_Wells
+Allen_Johnson
+Allenby_Bridge
+Allensbach
+Allentown,_Pennsylvania
+Alliance_'90/The_Greens
+Alloa_Athletic_F.C.
+Ally_McCoist
+Almere
+Aloÿs_Nizigama
+Alpay_Özalan
+Alpine_skiing_at_the_1994_Winter_Olympics
+Alsace
+Altenberg,_Germany
+Altin_Haxhi
+Altin_Rraklli
+Amalgamated_Roadstone_Corporation
+Amanda_Coetzer
+Amara_Essy
+Amarillo,_Texas
+Amazon_River
+American_Airlines
+American_Broadcasting_Company
+American_Civil_War
+American_Depositary_Receipt
+American_League
+American_League_Central
+American_League_East
+American_League_West
+American_Stock_Exchange
+American_University
+American_Veterinary_Medical_Association
+American_studies
+Amica_Wronki
+Amman
+Amnesty_International
+Amoco
+Amr_Moussa
+Amr_Shabana
+Amsterdam
+Amstetten,_Lower_Austria
+Amtrak
+Amy_Frazier
+An-Nahar
+Ana_Fidelia_Quirot
+Anaheim_Ducks
+Anatolia
+Anchorage,_Alaska
+Anders_Forsbrand
+Andhra_Pradesh
+Andre_Agassi
+Andre_Markgraaff
+Andre_Snyman
+Andrea_Collinelli
+Andrea_Ferrigato
+Andrea_Gaudenzi
+Andrea_Giaconi
+Andrea_Glass
+Andreas_Andersson
+Andreas_Goldberger
+Andreas_Heraf
+Andreas_Herzog
+Andreas_Kappes
+Andreas_Köpke
+Andreas_Möller
+Andreas_Ogris
+Andreas_Seelig
+Andreas_Thom
+Andreas_Zeyer
+Andrei_Chesnokov
+Andrei_Kanchelskis
+Andrei_Olhovskiy
+Andrei_Pavel
+Andrei_Tchmil
+Andrew_Caddick
+Andrew_Coltart
+Andrew_Hudson
+Andrew_Magee
+Andrew_Mehrtens
+Andrew_Symonds
+Andrew_Wakefield
+Andriy_Medvedev
+Andruw_Jones
+André_Joubert
+André_Ribeiro
+André_Trulsen
+Andrés_Galarraga
+Andy_Benes
+Andy_Etchebarren
+Andy_Flower
+Andy_Goram
+Andy_Hinchcliffe
+Andy_Melville
+Andy_Pettitte
+Andy_Sinton
+Andy_Townsend
+Andy_Whittall
+Ange-Félix_Patassé
+Angel_Miranda
+Anghel_Iordănescu
+Anglicanism
+Anglo-Welsh_Cup
+Angola
+Angola_national_football_team
+Angélica_Gavaldón
+Anhui
+Anil_Kumble
+Anita_Gradin
+Anita_Wachter
+Anja_Rücker
+Ankara
+Anke_Huber
+Ann_Grossman
+Anna_Kournikova
+Anne-Gaëlle_Sidot
+Anne_Boleyn
+Annemari_Sandell-Hyvärinen
+Annemarie_Jorritsma
+Annett_Neumann
+Antara_(news_agency)
+Anthony_Bancarel
+Anthony_Gobert
+Anthony_Quayle
+Anthony_Raine_Barker
+Anthony_Sullivan
+Anthony_Washington
+Anthuan_Maybank
+Antioquia_Department
+Anto_Drobnjak
+Antoine_Kombouaré
+Antoine_Lahad
+Anton_Bruckner
+Anton_Doboş
+Anton_Pfeffer
+Anton_Shantyr
+Antonella_Bellutti
+Antonio_Esposito
+Antonio_Quarracino
+Antonio_Tartaglia
+Antony_Marlow
+Antwerp
+António_Folha
+António_Luís_Alves_Ribeiro_Oliveira
+Anyang_LG_Cheetahs
+Apple_Inc.
+Appleton,_Wisconsin
+Aqaba
+Arab_Contractors_(company)
+Arab_World
+Arab_citizens_of_Israel
+Arab_people
+Arabic_language
+Arad,_Romania
+Arantxa_Parra_Santonja
+Arantxa_Sánchez_Vicario
+Aravinda_de_Silva
+Arbil
+Arbroath_F.C.
+Arctic
+Ards_F.C.
+Argentina
+Ari-Pekka_Nikkola
+Arif_Erdem
+Arizona
+Arizona_Cardinals
+Arizona_State_Sun_Devils_football
+Arjan_Xhumba
+Arjuna_Ranatunga
+Arkadiusz_Bąk
+Arkansas
+Arkansas_State_Police
+Arlen_Specter
+Arlington,_Texas
+Armando_Reynoso
+Armen_Martirosyan_(athlete)
+Armenia
+Armenia_national_football_team
+Arminia_Bielefeld
+Arnaud_Boetsch
+Arnold_Rüütel
+Aron_Winter
+Arrigo_Sacchi
+Arrows
+Arsenal_F.C.
+Art_Howe
+Arthur_Levitt
+Arthur_Numan
+Artur_Jorge_(footballer)
+Artur_Lekbello
+Arundel
+Aryan_race
+As-Safir
+Asanka_Gurusinha
+Ascot_Racecourse
+Ashford_Town_F.C._(Kent)
+Ashia_Hansen
+Ashoknagar
+Ashta
+Asia
+Asif_Ali_Zardari
+Asif_Mujtaba
+Aslan_Maskhadov
+Asmara
+Association_for_Relations_Across_the_Taiwan_Straits
+Association_of_Tennis_Professionals
+Associação_Desportiva_Vasco_da_Gama
+Associação_Portuguesa_de_Desportos
+Aston_Villa_F.C.
+Astrid_Kumbernuss
+Ata-ur-Rehman
+Atalanta_B.C.
+Athens
+Athens_Metro
+Athletic_Bilbao
+Atlanta
+Atlanta_Braves
+Atlanta_Falcons
+Atlanta_Hawks
+Atlante_F.C.
+Atlantic_City,_New_Jersey
+Atlantic_Division_(NBA)
+Atlantic_Division_(NHL)
+Atlantic_Highlands,_New_Jersey
+Atlético_Madrid
+Ato_Boldon
+Auchan
+Auckland
+Audi
+Aung_San
+Aung_San_Suu_Kyi
+Australia
+Australia_Davis_Cup_team
+Australia_national_cricket_team
+Australia_national_rugby_union_team
+Australian_Capital_Territory
+Australian_Democrats
+Australian_Football_League
+Australian_Greens
+Australian_Labor_Party
+Australian_Open
+Australian_people
+Australian_rules_football
+Austria
+Austria_Fed_Cup_team
+Austria_national_football_team
+Austria_national_under-21_football_team
+Austrian_Empire
+Austrians
+Avalon,_New_Jersey
+Avianca
+Avigdor_Kahalani
+Axel_Schulz
+Axis_Bank
+Ayodhya
+Ayr_United_F.C.
+Azad_Kashmir
+Azerbaijan
+Azerbaijan_national_football_team
+BBC
+BNP_Paribas
+BP
+BSC_Young_Boys
+BVSC_Budapest
+Ba'ath_Party
+Babel_(newspaper)
+Baburam_Bhattarai
+Bachirou_Salou
+Baghdad
+Bagram
+Bahrain
+Bahía_Blanca
+Bailundo
+Baja_California
+Baku
+Bali
+Balkan_Bulgarian_Airlines
+Balkans
+Bally_Shoe
+Ballybunion
+Balmain_Tigers
+Balochistan,_Pakistan
+Baltimore
+Baltimore_Orioles
+Baltimore_Ravens
+Bancomext
+Bandundu
+Bangkok
+Bangladesh
+Bangladesh_Awami_League
+Bangladesh_Nationalist_Party
+Bangui
+Banharn_Silpa-archa
+Banja_Luka
+Bank_Indonesia
+Bank_One_Corporation
+Bank_of_Canada
+Bank_of_Finland
+Bank_of_Israel
+Bank_of_Japan
+Bank_of_Mexico
+Bank_of_New_Zealand
+Bank_of_Spain
+Banque_de_France
+Barangay_Ginebra_Kings
+Barbara_Paulus
+Barbara_Rittner
+Barbara_Schett
+Barbarian_F.C.
+Barcelona
+Barentsburg
+Barnet_F.C.
+Barnsley_F.C.
+Barrick_Gold
+Barry,_Vale_of_Glamorgan
+Barry_Bonds
+Barry_Lane
+Barry_Larkin
+Bart_Voskamp
+Barvikha
+Basarab_Panduru
+Basque_Country_(autonomous_community)
+Basque_Country_(greater_region)
+Batajnica
+Batasuna
+Bath_Rugby
+Baudouin_of_Belgium
+Bavaria
+Bayer_04_Leverkusen
+Bayer_Giants_Leverkusen
+Bayer_HealthCare_Pharmaceuticals
+Beatrix_of_the_Netherlands
+Bedouin
+Beijing
+Beirut
+Beirut_Stock_Exchange
+Beitar_Jerusalem_F.C.
+Belarus
+Belarus_national_football_team
+Belfast
+Belga_(news_agency)
+Belgium
+Belgium_Fed_Cup_team
+Belgium_national_football_team
+Belgrade
+BellSouth
+Bellerive_Oval
+Belém
+Benazir_Bhutto
+Benedetto_Santapaola
+Benelux
+Benetton_Formula
+Benghazi
+Benin_national_football_team
+Benito_Mussolini
+Benito_Santiago
+Benjamin_Netanyahu
+Benoît_Cauet
+Benson_Koech
+Bergen
+Berkshire_Hathaway
+Berlin
+Berlin_Tegel_Airport
+Berlin_Tempelhof_Airport
+Bermuda
+Bern
+Bernama
+Bernard_Barmasai
+Bernard_Collomb
+Bernard_Ingham
+Bernard_Lama
+Bernard_Tapie
+Bernd_Karbacher
+Bernhard_Langer
+Berovo
+Bert_Konterman
+Bertelsmann
+Berti_Vogts
+Bertrand_Crasson
+Berwick_Rangers_F.C.
+Bessemer_Venture_Partners
+Best_Products
+Beth_Daniel
+Bethlehem
+Be��chatów
+Beşiktaş_J.K.
+Bharat_Ratna
+Bharatiya_Janata_Party
+Bhavnagar
+Biathlon_World_Cup
+Bible
+Bilbao
+Bild
+Biljana_Plavšić
+Bill_Birch
+Bill_Clinton
+Bill_Russell_(baseball)
+Billa_(supermarket)
+Billie_Jean_King
+Billings,_Montana
+Billy_Andrade
+Billy_Ashley
+Billy_Davies
+Billy_Dodds
+Billy_Mayfair
+Bima_Sakti
+Biogen_Idec
+Birendra_of_Nepal
+Birmingham
+Birmingham_City_F.C.
+Bistriţa
+Bixente_Lizarazu
+Black_Sea
+Blackburn_Rovers_F.C.
+Blackpool_F.C.
+Blaise_Compaoré
+Blantyre,_Malawi
+Bledar_Kola
+Blendi_Nallbani
+Blida_Province
+Bnei_Yehuda_Tel_Aviv_F.C.
+Board_of_Control_for_Cricket_in_India
+Boavista_F.C.
+Bob_Brett
+Bob_Dole
+Bob_Dwyer
+Bob_Estes
+Bob_Halverson
+Bob_Kennedy_(athlete)
+Bob_May_(golfer)
+Bob_Wickman
+Bob_Willis
+Bob_Wollek
+Bob_Woolmer
+Bobbie_Goulding
+Bobby_Bonilla
+Bobby_Charlton
+Bobby_Rahal
+Bobby_Robson
+Bobby_Valentine
+Boca_Juniors
+Boddington_Gold_Mine
+Boeing
+Bogdan_Stelea
+Bogotá
+Bogra
+Bohdan_Ulihrach
+Bohemians_1905
+Boland_cricket_team
+Bolivia
+Bolloré
+Bologna_F.C._1909
+Bolton_Wanderers_F.C.
+Bonn
+Bordeaux
+Boreham_Wood_F.C.
+Boris_Becker
+Boris_Yeltsin
+Borodino_(village),_Mozhaysky_District,_Moscow_Oblast
+Boroughmuir_RFC
+Borussia_Dortmund
+Borussia_Mönchengladbach
+Borussia_Neunkirchen
+Bosnia
+Bosnia_(region)
+Boston
+Boston_Bruins
+Boston_Celtics
+Boston_Red_Sox
+Boston_United_F.C.
+Botafogo_de_Futebol_e_Regatas
+Botswana
+Botswana_national_football_team
+Boulder,_Colorado
+Boutros_Boutros-Ghali
+Boutros_Harb
+Bovine_spongiform_encephalopathy
+Boxer_Protocol
+Brad_Ausmus
+Brad_Bryant
+Brad_Clontz
+Brad_Hogg
+Bradford_Bulls
+Bradford_City_A.F.C.
+Bradford_Vaughan
+Bradley_Hughes_(golfer)
+Brady_Anderson
+Brahim_Lahlafi
+Brasília
+Bratislava
+Brazil
+Brazil_national_football_team
+Brazilian_Football_Confederation
+Bre-X
+Brechin_City_F.C.
+Breda
+Breisach
+Bremen_Airport
+Brenda_Schultz-McCarthy
+Brenden_Pappas
+Brent_Mayne
+Brentford_F.C.
+Brett_Liddle
+Brett_Martin
+Brian_Burke_(ice_hockey)
+Brian_Currin
+Brian_De_Palma
+Brian_Giles
+Brian_Henninger
+Brian_Lara
+Brian_Laudrup
+Brian_McMillan
+Brian_McRae
+Brian_Shimer
+Brian_Wellman
+Bridgend_Ravens
+Brighton_&_Hove_Albion_F.C.
+Brigita_Bukovec
+Brisbane
+Brisbane_Broncos
+Brisbane_Lions
+Bristol
+Bristol_City_F.C.
+Bristol_Rovers_F.C.
+Bristol_Rugby
+British_Airways
+British_Columbia
+British_Heart_Foundation
+British_Jews
+British_Land
+British_Masters
+British_Universities_cricket_team
+British_West_Indies
+British_people
+Brno
+Brown_Deer_Park_Golf_Course
+Bruce_Dyer
+Bruce_Grobbelaar
+Brunei
+Bruno_Risi
+Bruno_Rodriguez
+Bruno_Thiry
+Bruny_Surin
+Brush_Engineered_Materials
+Brussels
+Bryan_Herta
+Bryan_Robson
+Bucharest
+Budapest
+Budapest_Honvéd_FC
+Budapest_Stock_Exchange
+Buddy_Groom
+Buenaventura,_Valle_del_Cauca
+Buenos_Aires
+Buenos_Aires_Province
+Buffalo_Bills
+Buffalo_Sabres
+Building_(magazine)
+Bukavu
+Bulgaria
+Bulgaria_national_football_team
+Bulgaria_national_under-21_football_team
+Bulgarians
+Bunge_Limited
+Bureau_Veritas
+Burhanuddin_Rabbani
+Burkhard_Reich
+Burkina_Faso
+Burma
+Burnley_F.C.
+Burundi
+Burundi_national_football_team
+Bury_F.C.
+Business_Recorder
+Butch_Harmon
+Buñol
+Byron_Black
+Béja
+Béla_Markó
+C.D._Guadalajara
+C.F._Monterrey
+C.F._Os_Belenenses
+C.F._Pachuca
+C._Rangarajan
+CB_Estudiantes
+CB_Sevilla
+CD_Tenerife
+CF_Extremadura
+CITIC_Pacific
+CNBC
+CNN
+CONCACAF_Champions_League
+CRH_plc
+CR_Vasco_da_Gama
+CSP_Limoges
+CS_Jiul_Petroşani
+Cabinet_(government)
+Caerphilly_RFC
+Caesarea
+Cagliari_Calcio
+Cairo
+Cal_Eldred
+Cal_Ripken,_Jr.
+Calgary
+Calgary_Flames
+California
+Caltex
+Calvin_Davis
+Cambodia
+Cambridge,_Massachusetts
+Cambridge_United_F.C.
+Camden,_Arkansas
+Camelot,_Chesapeake,_Virginia
+Cameroon
+Cameroon_national_football_team
+Camilla_Martin
+Canada
+Canadian_Grain_Commission
+Canadian_Prairies
+Canadian_Wheat_Board
+Canberra
+Canberra_Raiders
+Canopic_jar
+Canterbury-Bankstown_Bulldogs
+Cape_Town
+Caracas
+Cardiff
+Cardiff_City_F.C.
+Cardiff_RFC
+Cargill
+Caribbean
+Carl-Uwe_Steeb
+Carl_Fogarty
+Carl_Hooper
+Carl_Lewis
+Carl_Mason
+Carl_Suneson
+Carla_Sacramento
+Carlisle_United_F.C.
+Carlo_Ancelotti
+Carlo_Checchinato
+Carlos_Bianchi
+Carlos_Calado
+Carlos_Checa
+Carlos_Costa
+Carlos_Delgado
+Carlos_Filipe_Ximenes_Belo
+Carlos_Moyá
+Carlos_Pavón
+Carlos_Sainz
+Carlos_Secretário
+Carlsbad,_California
+Carlsberg_Group
+Carlton_Football_Club
+Carole_Montillet
+Carolina_Panthers
+Caroline,_Princess_of_Hanover
+Carroll_A._Campbell,_Jr.
+Carsten_Ramelow
+Carsten_Wolf
+Cartagena,_Colombia
+Casa_de_Nariño
+Casey_FitzRandolph
+Castle_Park_Cricket_Ground
+Castleford_Tigers
+Catalonia
+Catania
+Catholic_Church
+Catholicism
+Cathy_Freeman
+CeBIT
+Cecil_Mamiit
+Celsius
+Celta_de_Vigo
+Celtic_F.C.
+Central_African_Republic
+Central_African_Republic_national_football_team
+Central_Asia
+Central_Bank_of_Brazil
+Central_Clinical_Hospital
+Central_Division_(NBA)
+Central_Division_(NHL)
+Central_European_Free_Trade_Agreement
+Central_Narcotics_Bureau
+Cercle_Brugge_K.S.V.
+Chabab_Massira
+Chad_Curtis
+Chad_Ogea
+Challenge_Cup
+Chaminda_Vaas
+Chanda_Rubin
+Chandigarh
+Chandra_Sturrup
+Chandrika_Kumaratunga
+Changi_Prison
+Channel_2_(Israel)
+Channel_Islands
+Charleroi
+Charles,_Prince_of_Wales
+Charles_Austin
+Charles_Nagy
+Charles_Taylor_(Liberia)
+Charles_VI,_Holy_Roman_Emperor
+Charleston,_South_Carolina
+Charleston,_West_Virginia
+Charlotte_Bobcats
+Charlton_Athletic_F.C.
+Charmaine_Crooks
+Chatichai_Choonhavan
+Chavakacheri
+Chavalit_Yongchaiyudh
+Cheah_Soon_Kit
+Chechen_people
+Chechnya
+Chelsea_Clinton
+Chelsea_F.C.
+Chenab_River
+Cherbourg-Octeville
+Chesapeake,_Virginia
+Chester-le-Street
+Chester_City_F.C.
+Chesterfield
+Chesterfield_F.C.
+Chevron_Corporation
+Chiapas
+Chicago
+Chicago_Bears
+Chicago_Blackhawks
+Chicago_Board_Options_Exchange
+Chicago_Board_of_Trade
+Chicago_Bulls
+Chicago_Cubs
+Chicago_Mercantile_Exchange
+Chicago_Stock_Exchange
+Chicago_White_Sox
+Chihuahua,_Chihuahua
+Chile
+Chile_national_football_team
+Chili_Davis
+Chillicothe,_Ohio
+Chilpancingo
+China_Daily
+China_PR_national_football_team
+China_Steel
+Chinese_language
+Chinese_people
+Chipper_Jones
+Chiquinho_Conde
+Chiron_Corporation
+Chişinău
+Chongqing
+Chorzów
+Chris_Adams_(cricketer)
+Chris_Boardman
+Chris_Cairns
+Chris_Harris_(cricketer)
+Chris_Hay
+Chris_Hoiles
+Chris_Lewis_(cricketer)
+Chris_Patten,_Baron_Patten_of_Barnes
+Chris_Powell
+Chris_Sutton
+Chris_Walker_(squash_player)
+Chris_Witty
+Chris_Woodruff
+Christchurch
+Christian_Blanc
+Christian_Cullen
+Christian_Cévaër
+Christian_Karembeu
+Christian_Ruud
+Christian_Ruuttu
+Christian_Springer
+Christian_Ziege
+Christian_theology
+Christine_Magnusson
+Christine_Wachtel
+Christophe_Bonvin
+Christophe_Ohrel
+Christopher_Reeve
+Christopher_Wreh
+Chrysler_Classic_of_Tucson
+Chryste_Gaines
+Chua_Jui_Meng
+Chuck_Adams
+Chuck_Finley
+Cincinnati
+Cincinnati_Bengals
+Cincinnati_Reds
+Circuit_de_Spa-Francorchamps
+Ciriaco_Sforza
+Civil_Aviation_Administration_of_China
+Civil_Aviation_Authority_(United_Kingdom)
+Clarence_Park,_Weston-super-Mare
+Clarence_Rose
+Clarence_Seedorf
+Clarence_Woolmer
+Clarksburg,_West_Virginia
+Claude_Lelouch
+Claude_Makélélé
+Claudio_Suárez
+Cleveland
+Cleveland_Cavaliers
+Cleveland_Indians
+Cliftonville_F.C.
+Clinton_Whitelaw
+Clive_Lloyd
+Club_América
+Club_Atlas
+Club_Atlético_Banfield
+Club_Atlético_Huracán
+Club_Atlético_Independiente
+Club_Atlético_Lanús
+Club_Atlético_Platense
+Club_Atlético_River_Plate
+Club_Atlético_Vélez_Sársfield
+Club_Brugge_K.V.
+Club_Celaya
+Club_León
+Club_Necaxa
+Club_Santos_Laguna
+Club_Universidad_Nacional
+Clube_Atlético_Bragantino
+Clube_Atlético_Mineiro
+Clube_Atlético_Paranaense
+Clube_de_Regatas_do_Flamengo
+Clyde_F.C.
+Clydebank_F.C.
+Coalition_(Australia)
+Coast_guard
+Cochabamba
+Cocker_Spaniel
+Codelco
+Codos
+Colchester
+Colchester_United_F.C.
+Coleraine_F.C.
+Colin_Calderwood
+Colin_Cameron_(footballer)
+Colin_Edwards
+Colin_Hendry
+Colin_Jackson
+Colin_McRae
+Colin_Montgomerie
+Collingtree_Park
+Collingwood_Football_Club
+Cologne
+Colombia
+Colombian_Liberal_Party
+Colombo
+Colorado
+Colorado_Avalanche
+Colorado_Rockies
+Columbia_University
+Columbus,_Ohio
+Commack,_New_York
+Commerce_Bancshares
+Commission_on_Presidential_Debates
+Commonwealth_Games
+Commonwealth_of_Nations
+Communist_Party_of_China
+Communist_party
+Community_of_Madrid
+Competition_Commission_(United_Kingdom)
+Comprehensive_Nuclear-Test-Ban_Treaty
+Conakry
+Conchita_Martínez
+Confederate_States_of_America
+Confederation_of_African_Football
+Confederation_of_British_Industry
+Congo_DR_national_football_team
+Congo_national_football_team
+Conservative_Party_(UK)
+Constand_Viljoen
+Constantin_Gâlcă
+Constanţa
+Construction_Industry_Council
+Continental_AG
+Copenhagen
+Coppa_Italia
+Corey_Pavin
+Corina_Morariu
+Coritiba_Foot_Ball_Club
+Cornelius_Euser
+Cornell_Brown
+Corporación_Venezolana_de_Guayana
+Corsica
+Cosenza_Calcio_1914
+Cosmin_Contra
+Costa_Rica
+Costa_Rica_national_football_team
+Costain_Group
+Costantino_Rocca
+Costas_Simitis
+Council_of_Europe
+County_Antrim
+County_Championship
+County_Cricket_Ground,_Bristol
+County_Cricket_Ground,_Hove
+County_Tipperary
+Court_of_Cassation_(France)
+Courtney_Walsh
+Coventry_City_F.C.
+Cowdenbeath_F.C.
+Craig_Brown_(footballer)
+Craig_Burley
+Craig_Dowd
+Craig_Evans
+Craig_MacLean
+Craig_Matthews
+Craig_Parry
+Craig_Quinnell
+Craig_Spearman
+Craig_Stadler
+Craig_Wishart
+Crawley
+Credit_Suisse
+Creedmoor,_North_Carolina
+Creutzfeldt–Jakob_disease
+Crewe_Alexandra_F.C.
+Criciúma_Esporte_Clube
+Cricket_World_Cup
+Criminal_Investigation_Department
+Croatia
+Croatia_Davis_Cup_team
+Croatia_national_football_team
+Crohn's_disease
+Cronulla-Sutherland_Sharks
+Crossfire_(film)
+Crusaders_F.C.
+Cruz_Azul
+Cruzeiro_Esporte_Clube
+Crystal_Palace,_London
+Crystal_Palace_F.C.
+Crédit_Agricole_(cycling_team)
+Csepel_SC
+Cuauhtémoc_Blanco
+Cuba
+Cuba_national_football_team
+Culpeper,_Virginia
+Curitiba
+Curragh_Racecourse
+Currie_RFC
+Curtis_Fleming
+Curtly_Ambrose
+Cy_Young_Award
+Cycle_Collstrop
+Cynthia_McKinney
+Cyprus
+Cyprus_national_football_team
+Czech_National_Bank
+Czech_Republic
+Czech_Republic_Fed_Cup_team
+Czech_Republic_men's_national_ice_hockey_team
+Czech_Republic_national_football_team
+Czechoslovakia
+Czechs
+Cédric_Pioline
+Côte_d'Ivoire
+D._A._Weibring
+DAX
+DFB-Pokal
+Daewoo
+Daily_News_(New_York)
+Dakar
+Dale_McIntosh
+Dalian
+Dallas
+Dallas_Cowboys
+Dallas_Mavericks
+Dallas_Stars
+Dally_Randriantefy
+Dalmatian_(dog)
+Damascus
+Damian_Lynch
+Damion_Easley
+Damon_Hill
+Dampier,_Western_Australia
+Dan_Crowley
+Dan_Glickman
+Dan_Jenson
+Dan_Petrescu
+Dana_Rohrabacher
+Daniel_Andersson_(footballer_born_1977)
+Daniel_Bravo
+Daniel_Chopra
+Daniel_Ducruet
+Daniel_Herbert
+Daniel_Komen
+Daniel_Nestor
+Daniel_Prodan
+Daniel_Vacek
+Daniel_da_Cruz_Carvalho
+Daniele_Carnasciali
+Danilo_Hondo
+Danny_Blind
+Danny_Tartabull
+Dante_Bichette
+Darius_Milhaud
+Dariusz_Rosati
+Dariusz_Wosz
+Darlington_F.C.
+Darren_Anderton
+Darren_Bragg
+Darren_Dreifort
+Darren_Eadie
+Darren_Garforth
+Darren_Gough
+Darren_Jackson
+Darren_Lehmann
+Darryl_Powell
+Darryl_Strawberry
+Darryn_Hill
+Daryll_Cullinan
+Dave_Barr_(golfer)
+Dave_Gilbert_(cricketer)
+Dave_McPherson_(footballer)
+Dave_Nilsson
+Dave_Richardson
+Dave_Telgheder
+Dave_Wilson_(rugby_union)
+Davey_Johnson
+David
+David_Batty
+David_Boon
+David_Brabham
+David_Byas
+David_Campese
+David_Carter_(golfer)
+David_Coulthard
+David_Elleray
+David_Evans_(squash_player)
+David_Giffin
+David_Gilford
+David_Ginola
+David_Howell_(golfer)
+David_Hulse_(baseball)
+David_J_Russell
+David_Kelly_(footballer)
+David_Levy_(Israeli_politician)
+David_MacEachern
+David_Millns
+David_Peleg
+David_Platt_(footballer)
+David_Prinosil
+David_Richards_(racing)
+David_Rikl
+David_Rowson
+David_Segui
+David_Sesa
+David_Wheaton
+David_Zitelli
+David_Škoch
+Davidson_Ezinwa
+Davis_Cup
+Davis_Kamoga
+Davis_Love_III
+Davor_Šuker
+Dawn_(newspaper)
+Dayton,_Ohio
+Dayton_Agreement
+De_Graafschap
+Dead_Sea
+Dean_Gorré
+Dean_Headley
+Dean_Holdsworth
+Dean_Jones_(cricketer)
+Dean_Palmer
+Dean_Richards_(rugby_union)
+Dean_Saunders
+Dean_Sturridge
+Dean_Ward
+Dean_Windass
+Debbie_Graham
+Deborah_Compagnoni
+Debreceni_VSC
+Decatur,_Illinois
+Deere_&_Company
+Dejan_Koturović
+Dejan_Savićević
+Dejan_Stefanović
+Del_Harris_(squash_player)
+Delfino_Pescara_1936
+Delino_DeShields
+Democracy_Wall
+Democratic_Karen_Buddhist_Army
+Democratic_Left_Alliance
+Democratic_National_Convention
+Democratic_Party_(Hong_Kong)
+Democratic_Party_(United_States)
+Democratic_Party_of_Iranian_Kurdistan
+Democratic_Republic_of_the_Congo
+Democratic_Union_of_Hungarians_in_Romania
+Deng_Xiaoping
+Denis_Irwin
+Denis_Streak
+Denmark
+Dennis_Bergkamp
+Dennis_Lillee
+Dennis_Mitchell
+Dennis_Ross
+Denny_Neagle
+Denver
+Denver_Broncos
+Denver_Nuggets
+Deon_Hemmings
+Department_for_International_Development
+Deportivo_Español
+Deportivo_Toluca_F.C.
+Deportivo_de_La_Coruña
+Der_Spiegel
+Derby_County_F.C.
+Derbyshire_County_Cricket_Club
+Derek_Crookes
+Derek_Jeter
+Derek_Mills
+Derek_Ringer
+Derek_Ryan
+Derrick_Adkins
+Derrick_Cooper
+Des_Moines,_Iowa
+Des_Smyth
+Des_Terblanche
+Desmond_Tutu
+Desvonde_Botes
+Detroit
+Detroit_Lions
+Detroit_Pistons
+Detroit_Red_Wings
+Detroit_Tigers
+Deutsche_Bahn
+Deutsche_Bundesbank
+Deutsche_Mark
+Devon_White_(baseball)
+Dewas
+Dhahran
+Dhaka
+Dhaka_Stock_Exchange
+Dhar
+Dia_(supermarket_chain)
+Diana,_Princess_of_Wales
+Dianne_Feinstein
+Diario_16
+Dick_Gephardt
+Dick_Morris
+Dick_Schreuder
+Dick_Spring
+Didier_Deschamps
+Diego_Borrego
+Diego_Domínguez
+Dieter_Eilts
+Dieter_Ramusch
+Dieter_Thoma
+Dietmar_Beiersdorfer
+Dietmar_Hirsch
+Dietmar_Kühbauer
+Dili
+Dimas_Teixeira
+Dinamina
+Dirk_Coetzee
+Dirk_Medved
+Dirk_Wiese
+Discovery_Channel_Pro_Cycling_Team
+Divaina
+Dmitri_Dashinski
+Dmitri_Markov
+Dnevni_Avaz
+Doberman_Pinscher
+Doboj
+Dodge_City,_Kansas
+Doetinchem
+Doku_Zavgayev
+Dominic_Cork
+Dominic_Hewson
+Dominion_Bond_Rating_Service
+Dominique_Baratelli
+Dominique_Monami
+Don_McKinnon
+Don_Wengert
+Donald_Tsang
+Donaldson,_Lufkin_&_Jenrette
+Donato_Gama_da_Silva
+Doncaster_Cup
+Doncaster_Rovers_F.C.
+Dong_Jiong
+Donna_Andrews_(golfer)
+Donna_Weinbrecht
+Donne_Wall
+Donovan_Bailey
+Dore_Gold
+Dorinel_Munteanu
+Doug_Flach
+Doug_Young
+Dow_Chemical_Company
+Dow_Jones_Industrial_Average
+Dresden
+Dresden_Airport
+Dubai
+Dublin
+Ducati
+Ducati_Corse
+Duff_&_Phelps
+Duffy_Waldorf
+Duhok,_Iraq
+Duilio_Davino
+Duluth,_Minnesota
+Duma
+Dumbarton_F.C.
+Duncan_Ferguson
+Dundee_F.C.
+Dundee_United_F.C.
+Dunedin
+Dunfermline_Athletic_F.C.
+Dunnes_Stores
+Dunvant_RFC
+Durban
+Durham_County_Cricket_Club
+Durrës
+Dushanbe
+Dust_Bowl
+Dutch_people
+Dwight_Yorke
+Dynamo_Sports_Club
+Dzhokhar_Dudayev
+Dési_Bouterse
+Dănuţ_Lupu
+ETA
+Eamonn_Darcy
+East_Fife_F.C.
+East_Germany
+East_Java
+East_Jerusalem
+East_Kalimantan
+East_North_Central_States
+East_Stirlingshire_F.C.
+East_Timor
+Eau_Claire,_Wisconsin
+Ebbw_Vale_RFC
+Eberhard_Carl
+Economic_Community_of_West_African_States
+Economic_Community_of_West_African_States_Monitoring_Group
+Ecuador
+Ed_Vosberg
+Ed_de_Goey
+Eddie_Irvine
+Eddie_Murray
+Eddo_Brandes
+Edgar_Davids
+Edgar_Martínez
+Edgar_Rentería
+Edgbaston
+Edgbaston_Cricket_Ground
+Edinburgh
+Edmonton
+Edmonton_Oilers
+Eduard_Gritsun
+Eduardo_Romero
+Edvard_Grieg
+Edward_Dmytryk
+Edward_Said
+Edwin_Godee
+Edwin_Vurens
+Edwin_van_der_Sar
+Edwina_Currie
+Efan_Ekoku
+Efes_Pilsen_S.K.
+Egypt
+EgyptAir
+Egyptians
+Eindhoven
+Ejup_Ganić
+El_Al
+El_Mundo_(Spain)
+El_Nuevo_Diario
+El_País
+El_Salvador
+El_Salvador_national_football_team
+El_Watan
+Elakhbar
+Elena_Likhovtseva
+Elena_Makarova
+Elena_Pampoulova
+Eleusina
+Elf_Aquitaine
+Elgin_City_F.C.
+Elia_Kazan
+Eliyahu_Ben-Elissar
+Elizabeth,_New_Jersey
+Elizabeth_I_of_England
+Elizabeth_McIntyre
+Elland_Road
+Ellina_Zvereva
+Ellis_Burks
+Ellis_Park_Stadium
+Els_Callens
+Emanuele_Canonica
+Emil_Constantinescu
+Emiliano_Mondonico
+Emilio_Valle
+Emmanuel_Tetteh
+Empire_State_Building
+Empire_of_Brazil
+Empoli_F.C.
+En_Avant_de_Guingamp
+Endrio_Leoni
+Enfield_Town_F.C.
+England
+England_cricket_team
+England_national_football_team
+England_national_rugby_union_team
+English_language
+English_people
+Enrico_Chiesa
+Enrique_Alfaro
+Environment_Canada
+Enzo_Scifo
+Equipe_Ligier
+Equitas
+Eredivisie
+Erez
+Eric_Anthony
+Eric_Bergoust
+Eric_Davis_(baseball)
+Eric_Rush
+Eric_Thomas_(athlete)
+Eric_Wynalda
+Ericsson
+Erik_Breukink
+Erik_Dekker
+Erik_Hanson
+Erik_Zabel
+Eritrea
+Erjon_Bogdani
+Ernest_Faber
+Ernesto_Samper
+Ernie_Els
+Ervin_Fakaj
+Erzincan
+Eschen
+Espen_Bredesen
+Esporte_Clube_Bahia
+Esporte_Clube_Flamengo
+Esporte_Clube_Juventude
+Esporte_Clube_Vitória
+Essendon_Football_Club
+Essex
+Essex_County_Cricket_Club
+Essilor
+Estonia
+Estonia_national_football_team
+Estonian_Reform_Party
+Estudiantes_Tecos
+Estudiantes_de_La_Plata
+Eternit
+Ethiopia
+Ethiopia_national_football_team
+Etienne_Saqr
+Etruria
+Eugene_Emeralds
+Eugene_de_Kock
+Eugenio_Corini
+Eurodollar
+Euroleague_Basketball
+Euronext_Paris
+Europe
+Europe_1
+European_Commission
+European_Court_of_Human_Rights
+European_Cup_(athletics)
+European_Economic_Community
+European_Union
+Euroscepticism
+Eurostat
+Eurotunnel
+Evelyne_Leu
+Everton_F.C.
+Ewald_Brenner
+Exeter_City_F.C.
+Expansión
+Exxon
+ExxonMobil
+Ezer_Weizman
+F.C._Hansa_Rostock
+F.C._Internazionale_Milano
+F.C._Porto
+F._W._de_Klerk
+FA_Cup
+FC_Aarau
+FC_Alania_Vladikavkaz
+FC_Baltika_Kaliningrad
+FC_Baník_Ostrava
+FC_Barcelona
+FC_Barcelona_Bàsquet
+FC_Bayern_Munich
+FC_CSKA_Kyiv
+FC_Ceahlăul_Piatra_Neamţ
+FC_Chornomorets_Odesa
+FC_Dinamo_Batumi
+FC_Dinamo_Bucureşti
+FC_Dnipro_Dnipropetrovsk
+FC_Dynamo_Kyiv
+FC_Dynamo_Moscow
+FC_Girondins_de_Bordeaux
+FC_Groningen
+FC_Hradec_Králové
+FC_Karpaty_Lviv
+FC_Kotayk_Abovian
+FC_Kremin_Kremenchuk
+FC_Krylia_Sovetov_Samara
+FC_Kryvbas_Kryvyi_Rih
+FC_Lada_Togliatti
+FC_Lausanne-Sport
+FC_Linz
+FC_Lokomotiv_Moscow
+FC_Lokomotiv_Nizhny_Novgorod
+FC_Lokomotíva_Košice
+FC_Lugano
+FC_Luzern
+FC_Metalurh_Zaporizhya
+FC_Metz
+FC_Nantes
+FC_Nitra
+FC_Nyva_Ternopil
+FC_Oţelul_Galaţi
+FC_Rapid_Bucureşti
+FC_Red_Bull_Salzburg
+FC_Rostov
+FC_Rotor_Volgograd
+FC_Schalke_04
+FC_Shakhtar_Donetsk
+FC_Sion
+FC_Slovan_Liberec
+FC_Spartak_Moscow
+FC_Spartak_Trnava
+FC_Sportul_Studenţesc_Bucureşti
+FC_St._Gallen
+FC_St._Pauli
+FC_Steaua_Bucureşti
+FC_Tekstilshchik_Kamyshin
+FC_Tiraspol
+FC_Tirol_Innsbruck
+FC_Torpedo_Moscow
+FC_Torpedo_Zaporizhya
+FC_Twente
+FC_Universitatea_Cluj
+FC_Universitatea_Craiova
+FC_Ural_Sverdlovsk_Oblast
+FC_Utrecht
+FC_Vaduz
+FC_Viktoria_Plzeň
+FC_Volendam
+FC_Volgograd
+FC_Vorskla_Poltava
+FC_Zbrojovka_Brno
+FC_Zenit_Saint_Petersburg
+FC_Zhemchuzhina-Sochi
+FDA_(trade_union)
+FIFA
+FIFA_World_Cup
+FIS_Alpine_Ski_World_Cup
+FIS_Freestyle_Skiing_World_Cup
+FIS_Ski_Jumping_World_Cup
+FK_Austria_Wien
+FK_Baumit_Jablonec
+FK_Borac_Čačak
+FK_Budućnost_Podgorica
+FK_DAC_1904_Dunajská_Streda
+FK_Drnovice
+FK_Dukla_Banská_Bystrica
+FK_Hajduk_Beograd
+FK_Hajduk_Kula
+FK_Inter_Bratislava
+FK_Kareda_Kaunas
+FK_Kikinda
+FK_Metalurg_Skopje
+FK_Partizan
+FK_Proleter_Zrenjanin
+FK_Rad
+FK_Rudar_Pljevlja
+FK_Rudar_Ugljevik
+FK_Sloboda_Tuzla
+FK_Sloga_Jugomagnat
+FK_Spartak_Zlatibor_Voda
+FK_Sutjeska_Foča
+FK_Sutjeska_Nikšić
+FK_Teplice
+FK_Viktoria_Žižkov
+FK_Vojvodina
+FK_Zemun
+FSC_Prykarpattya_Ivano-Frankivsk
+FTSE_100_Index
+Fabio_Baldato
+Fabio_Cannavaro
+Fabio_Capello
+Fabrizio_Mori
+Fabrizio_Ravanelli
+Fairmont,_West_Virginia
+Fairview,_Texas
+Falilat_Ogunkoya
+Falk_Balzer
+Falkirk_F.C.
+Fanie_de_Villiers
+Farmington_Hills,_Michigan
+Faroe_Islands
+Faroe_Islands_national_football_team
+Fatima_Yusuf
+Fatmir_Vata
+Fatos_Nano
+Faustino_Asprilla
+Fausto_Pizzi
+Fed_Cup
+Federal_Aviation_Administration
+Federal_Bureau_of_Investigation
+Federal_Open_Market_Committee
+Federal_Reserve_System
+Felipe_Lira
+Fenerbahçe_S.K.
+Ferdi_Vierklau
+Ferenc_Horváth
+Ferencvárosi_TC
+Fernanda_Ribeiro
+Fernando_Couto
+Fernando_Henrique_Cardoso
+Fernando_Hierro
+Fernando_Meligeni
+Fernando_Redondo
+Fernando_Sanz
+Ferrari
+Ferro_Carril_Oeste
+Feyenoord
+Fidel_Castro
+Fidel_V._Ramos
+Filip_De_Wilde
+Filip_Dewulf
+Filippo_Inzaghi
+Finance_minister
+Financial_Post
+Financial_Services_Authority
+Finland
+Finland_men's_national_ice_hockey_team
+Fiona_May
+First_Chicago_Bank
+First_Pacific
+Fita_Bayisa
+Fitzroy_Football_Club
+Five_Nations_XV
+Flachau
+Flag_of_the_United_States
+Flamurtari_Vlorë
+Flavio_Cotti
+Florence
+Florence_Masnada
+Florencia_Labat
+Florian_Maurice
+Florian_Rousseau
+Florian_Schwarthoff
+Florida_Marlins
+Florida_Panthers
+Florin_Prunea
+Fluminense_Football_Club
+Flushing_Meadows_–_Corona_Park
+Foindu
+Football_Association_of_Ireland
+Footwork_Arrows
+Ford_Escort
+Ford_Escort_(Europe)
+Ford_Motor_Company
+Ford_World_Rally_Team
+Foreign_Office_(Germany)
+Foreign_minister
+Forfar_Athletic_F.C.
+Formula_One
+Forrest_Gump
+Forsa_institute
+Fort_Lauderdale,_Florida
+Fort_Worth,_Texas
+Forth_Road_Bridge
+Fortitudo_Bologna
+Fortuna_Düsseldorf
+Fortuna_Sittard
+Fos-sur-Mer
+Fourth_World_Conference_on_Women
+Fox_Broadcasting_Company
+France
+France_Fed_Cup_team
+France_Soir
+France_Télécom
+France_national_football_team
+Francesca_Lubiani
+Francesco_Casagrande
+Francesco_Mazzariol
+Francesco_Toldo
+Francesco_Totti
+Francis_Agyepong
+Francis_Moreau
+Francisco_Clavet
+Francisco_Palencia
+Franco_Baresi
+Frank_Bruno
+Frank_Bunce
+Frank_Busemann
+Frank_Leboeuf
+Frank_Nobilo
+Frank_Rodriguez
+Frank_Thomas_(baseball,_born_1968)
+Frank_Vandenbroucke_(cyclist)
+Frank_de_Boer
+Franka_Dietzsch
+Frankfurt_Airport
+Frankfurt_Stock_Exchange
+Frankfurt_am_Main
+Frankie_Fredericks
+Franklin_D._Roosevelt
+Franz_Fischler
+Franz_Konrad
+Franziska_Schenk
+François_Pienaar
+Fraser,_Australian_Capital_Territory
+Fred_Couples
+Fred_Funk
+Fred_McGriff
+Fred_Trueman
+Frederick_Chiluba
+Fredi_Bobic
+Free_Democratic_Party_(Germany)
+Free_Democratic_Party_of_Switzerland
+Freedom_Front_Plus
+Freetown
+Freiburg_im_Breisgau
+Fremantle_Football_Club
+French_Democratic_Confederation_of_Labour
+French_Guiana
+French_Open
+French_Riviera
+Fribourg
+Fritz_van_Heerden
+Frode_Andresen
+Frédéric_Magné
+Frédéric_Peiremans
+Fulgencio_Batista
+Fulham_F.C.
+Fung_Permadi
+Fußball-Bundesliga
+Fédération_Syndicale_Unitaire
+Félicia_Ballanger
+Félix_Mantilla_Botella
+GKS_Bełchatów
+GKS_Katowice
+Gabon
+Gabon_national_football_team
+Gabriel_Batistuta
+Gabriel_Curuchet
+Gabriela_Sabatini
+Gabriela_Szabo
+Gabriele_Colombo
+Gail_Devers
+Gala_León_García
+Galanta
+Galatasaray_S.K._(football_team)
+Gale_Norton
+Galina_Malchugina
+Galo_Blanco
+Garamba_National_Park
+Gareth_Farrelly
+Garhi_Habibullah
+Garret_Anderson
+Garrett_Hines
+Garry_Galley
+Garry_Pagel
+Gartner
+Gary_Breen
+Gary_DiSarcina
+Gary_Emerson
+Gary_Kelly_(footballer_born_1974)
+Gary_Kirsten
+Gary_McAllister
+Gary_Neiwand
+Gary_Orr
+Gary_Sheffield
+Gary_Speed
+Gary_Teichmann
+Gaston_Taument
+Gatwick_Airport
+Gaza
+Gaza_Strip
+Gazeta_Wyborcza
+Gazprom
+Gdańsk
+Geelong_Football_Club
+Geir_Moen
+General_Administration_of_Customs
+General_Confederation_of_Labour_(France)
+General_Motors
+Geneva
+Genoa
+Genoa_C.F.C.
+Gente_(magazine)
+Geoff_Aunger
+Geoff_Marsh
+Geoffrey_Claeys
+Geoffrey_Fieger
+George_Duffield
+George_H._W._Bush
+George_Hincapie
+George_Joulwan
+George_Weah
+Georges_Simenon
+Georgia_(U.S._state)
+Georgia_(country)
+Georgios_Panagiotopoulos
+Georgiy_Mammadov
+Gerard_van_Velde
+Gerhard_Berger
+German_Open_(golf)
+German_Shepherd_Dog
+German_language
+Germans
+Germany
+Germany_Davis_Cup_team
+Germany_Fed_Cup_team
+Germany_men's_national_ice_hockey_team
+Germany_national_football_team
+Germán_Villa
+Geronimo
+Gerry_Britton
+Gert_Verheyen
+Gete_Wami
+Ghana
+Gheorghe_Craioveanu
+Gheorghe_Funar
+Gheorghe_Hagi
+Gheorghe_Popescu
+Gianfranco_Fini
+Gianfranco_Zola
+Gianluca_Pozzi
+Gianluca_Vialli
+Gianluigi_Lentini
+Gianni_Bugno
+Gianni_Versace
+Gigi_Fernández
+Gil_Vicente_F.C.
+Gil_de_Ferran
+Gilbert_Agius
+Gilbert_Gress
+Gilbert_Schaller
+Gilles_De_Bilde
+Gillian_Russell
+Gillingham_F.C.
+Giovanni_Lavaggi
+Giovanni_Lombardi
+Giovanni_Silva_de_Oliveira
+Giovanni_van_Bronckhorst
+Gisenyi
+Giuseppe_Garibaldi
+Gjon_Buzuku
+Glamorgan_County_Cricket_Club
+Glasgow
+Glen_Osborne
+Glenallen_Hill
+Glenavon_F.C.
+Glenn_Hoddle
+Glenn_McGrath
+Glentoran_F.C.
+Gloria_Pizzichini
+Gloucester_Rugby
+Gloucestershire_County_Cricket_Club
+God
+Goiás_Esporte_Clube
+Golan_Heights
+Gold_Coast_Chargers
+Gold_Coast_Titans
+Golden_State_Warriors
+Goldman_Sachs
+Goma
+Gong_Zhichao
+Goran_Ivanišević
+Gordon_Durie
+Gordon_Parsons
+Gorgona,_Colombia
+Gorleben
+Gouda
+Government_of_Russia
+Governor_of_South_Carolina
+Grace_Kelly
+Grace_Road
+Graeme_Hick
+Graeme_Obree
+Graham_Gooch
+Graham_Lloyd
+Graham_Thorpe
+Grand_Rapids,_Michigan
+Grand_Slam_(tennis)
+Grand_Slam_Cup
+Grande_Prairie
+Granma_(newspaper)
+Grant_Flower
+Grant_Stafford
+Grasshopper_Club_Zürich
+Grazer_AK
+Great_Britain_national_rugby_league_team
+Great_Hall_of_the_People
+Great_Lakes
+Greece
+Green_Bay_Packers
+Greenock_Morton_F.C.
+Greenville_Braves
+Greenwich_Mean_Time
+Greg_Blewett
+Greg_Chalmers
+Greg_Chappell
+Greg_Gagne_(baseball)
+Greg_Kraft
+Greg_Norman
+Greg_Rusedski
+Greg_Turner
+Gregor_Townsend
+Grimsby_Town_F.C.
+Griqualand_West
+Groningen_(city)
+Grozny
+Grupo_Santander
+Grupos_Antiterroristas_de_Liberación
+Grégory_Carraz
+Grêmio_Foot-Ball_Porto_Alegrense
+Guangdong
+Guangxi
+Guangzhou
+Guarani_Futebol_Clube
+Guardians_of_the_Cedars
+Guatemala
+Guernsey
+Guerrero
+Guido_Acklin
+Guido_Fulst
+Guilin
+Guillaume_Raoux
+Guillermo_Amor
+Guinea_national_football_team
+Gujar_Khan
+Gujarat
+Gulbuddin_Hekmatyar
+Gulf_War
+Gulf_of_Mexico
+Gunhild_Haugen
+Gunn_Margit_Andreassen
+Gunther_Schepens
+Guus_Hiddink
+Guy_Forget
+Guy_Hellers
+Guy_Whittall
+Guy_Whittingham
+Gwen_Torrence
+Győri_ETO_FC
+Gérard_de_Nooijer
+Górnik_Zabrze
+Günther_Huber
+H._D._Deve_Gowda
+HALO_Trust
+HINA
+HP_Enterprise_Services
+Haaretz
+Haarlem
+Habib_Boularès
+Hachette_Filipacchi_Médias
+Hafez_al-Assad
+Haile_Gebrselassie
+Hainan
+Haiti
+Hakan_Ünsal
+Hakan_Şükür
+Hakkâri
+Hakkâri_Province
+Hal_Morris
+Hal_Sutton
+Halifax_RLFC
+Hamas
+Hambrecht_&_Quist
+Hamburg
+Hamburg_Airport
+Hamburger_SV
+Hamid_Algabid
+Hamilton,_New_Zealand
+Hamilton_Academical_F.C.
+Hammerson
+Hampshire
+Hampshire_County_Cricket_Club
+Haneda_Airport
+Hang_Seng_Index
+Hangzhou
+Hanne_Haugland
+Hannes_Strydom
+Hanover
+Hans_Segers
+Hansie_Cronje
+Hanson_plc
+Hanwha_Eagles
+Hapoel_Be'er_Sheva_A.F.C.
+Hapoel_Haifa_F.C.
+Hapoel_Ironi_Rishon_LeZion_F.C.
+Hapoel_Jerusalem_F.C.
+Hapoel_Petah_Tikva_F.C.
+Hapoel_Tayibe_F.C.
+Hapoel_Tel_Aviv_F.C.
+Hapoel_Tzafririm_Holon_F.C.
+Harald_Spörl
+Harare
+Harlequin_F.C.
+Harlequins_Rugby_League
+Harleysville,_Pennsylvania
+Harold_Baines
+HarperCollins
+Harrison_Dillard
+Harry_Boland
+Harry_Decheiver
+Hartford,_Connecticut
+Hartford_Whalers
+Hartford_Wolf_Pack
+Hartlepool_United_F.C.
+Haruchika_Aoki
+Hasan_Muratović
+Hasely_Crawford
+Hassan_Abbas
+Hassan_II_of_Morocco
+Hassan_al-Turabi
+Hassania_Agadir
+Hastings_Banda
+Havana
+Havnar_Bóltfelag
+Havre,_Montana
+Hawaii
+Hawick_RFC
+Hawthorn_Football_Club
+Hay_Point,_Queensland
+Headingley
+Headingley_Stadium
+Heart_of_Midlothian_F.C.
+Heath_Streak
+Heathrow,_London
+Hebrew_University_of_Jerusalem
+Hebrew_language
+Hebron
+Hednesford_Town_F.C.
+Heidi_Zurbriggen
+Heidrun_oil_field
+Heike_Drechsler
+Heineken_International
+Heineken_Pilsener
+Heinz-Harald_Frentzen
+Helen_Clark
+Helena_Suková
+Heli_Rantanen
+Helibor
+Hellas_Verona_F.C.
+Helmut_Kohl
+Helsinki
+Hendrik_Dreekmann
+Hendro_Kartiko
+Hengelo
+Henri_Konan_Bédié
+Henrik_Larsson
+Henry_Campbell-Bannerman
+Henry_Honiball
+Henry_Hub
+Henry_VIII_of_England
+Herbert_Prohaska
+Herculez_Gomez
+Hereford_United_F.C.
+Herfried_Sabitzer
+Herman_Wijffels
+Hermann_Göring
+Hermawan_Susanto
+Hernán_Gumy
+Herschelle_Gibbs
+Hervé_de_Charette
+Hezbi_Islami
+Hezbollah
+Hibernian_F.C.
+Hicham_Arazi
+Hicham_El_Guerrouj
+Hidemichi_Tanaka
+Hideo_Nomo
+High_Court_(Hong_Kong)
+High_Plains_(United_States)
+Hilary_Lindh
+Hilde_Gerg
+Hilde_Synnøve_Lid
+Hillary_Rodham_Clinton
+Himalaya_Kingdom
+Himalayas
+Hindu_nationalism
+Hintsa_kaKhawuta
+Hiroaki_Morishima
+Hiroshi_Nanami
+Hiroshige_Yanagimoto
+Hobart
+Holding_company
+Hollywood
+Holy_See
+Holzgerlingen
+Honda
+Honda_Racing_Corporation
+Honduras
+Honduras_national_football_team
+Hong_Kong
+Hong_Kong_Economic_Times
+Hong_Kong_dollar
+Horst_Siegl
+Hosni_Mubarak
+House_of_Commons
+House_of_Orléans
+House_of_Representatives
+Houston
+Houston_Astros
+Houston_Cougars_football
+Houston_Rockets
+Houston_Texans
+Hove
+Howard_Wilkinson
+Howard_Wolpe
+Hristo_Stoichkov
+Hubei
+Hubert_Schösser
+Huddersfield_Town_F.C.
+Hugh_Baiocchi
+Huize_County
+Hull_City_A.F.C.
+Human_Rights_in_China_(organization)
+Humayun_Rashid_Choudhury
+Hun_Sen
+Hungarian_National_Bank
+Hungary
+Huntington,_West_Virginia
+Huntly_F.C.
+Hurricane_Edouard_(1996)
+Husaberg
+Husqvarna_Motorcycles
+Hussein_of_Jordan
+Hutnik_Kraków
+Hutu
+Hwang_Sun-Hong
+Hyderabad,_Sindh
+HypoVereinsbank
+Héctor_Camacho
+Hélder_Cristóvão
+Hércules_CF
+IBM
+IFK_Göteborg
+IFOR
+IK_Start
+ING_Group
+ISO_9000
+Iain_Pyman
+Ian_Botham
+Ian_Ferguson_(footballer_born_1967)
+Ian_Harte
+Ian_Harvey
+Ian_Healy
+Ian_Jones_(rugby_union)
+Ian_Lang,_Baron_Lang_of_Monkton
+Ian_McGeechan
+Ian_Rush
+Ian_Salisbury
+Ian_Woosnam
+Ian_Wright
+Iberian_Peninsula
+Ice-Cold_in_Alex
+Iceland
+Idris_I_of_Libya
+Idriss_Déby
+Ieng_Mouly
+Ieng_Sary
+Igor_Ivanov
+Igor_Kolyvanov
+Igor_Korneev
+Igor_Potapovich
+Igor_Shkvyrin
+Igor_Trandenkov
+Ijaz_Ahmed_(cricketer)
+Ilir_Shulku
+Ilke_Wyludda
+Illawarra_Steelers
+Illinois
+Iltalehti
+Imam_Utomo
+Imola
+Imran_Khan
+In-Nazzjon
+Ina-Yoko_Teutenberg
+Incheon
+Independence_Day_(film)
+Independent_Commission_Against_Corruption_(Hong_Kong)
+Independent_State_of_Croatia
+Inder_Kumar_Gujral
+India
+India_national_cricket_team
+Indian_Oil_Corporation
+Indiana
+Indiana_Pacers
+Indianapolis
+Indianapolis_Colts
+Indigenous_peoples_of_the_Americas
+Indonesia
+Indonesia_national_football_team
+Indonesian_Democratic_Party
+Indore
+Indrajit_Gupta
+Indre
+Industrial_Production_Index
+IndyCar_Series
+Inez_Turner
+Information_Technology_Association_of_America
+Information_Telegraph_Agency_of_Russia
+Inger_Miller
+Ingolstadt
+Inha_Babakova
+Inia,_Paphos
+Inkatha_Freedom_Party
+Insein_Prison
+Insein_Township
+Institute_for_Supply_Management
+Institute_of_Policy_Studies_(Singapore)
+Intellectual_Property_Owners_Association
+Intelligent_network
+Interfax
+Interferon_beta-1b
+Interior_ministry
+International_Association_of_Athletics_Federations
+International_Boxing_Association
+International_Committee_of_the_Red_Cross
+International_Confederation_of_Free_Trade_Unions
+International_Labour_Organization
+International_Monetary_Fund
+International_Petroleum_Exchange
+International_Union_of_Railways
+International_cricketers_of_South_African_origin
+Interstate_5
+Interstate_80
+Interstate_95
+Inverness
+Inverness_Caledonian_Thistle_F.C.
+Inverness_Thistle_F.C.
+Inzamam-ul-Haq
+Inés_Gorrochategui
+Ion_Iliescu
+Ionel_Dănciulescu
+Ipsos
+Ipswich_Town_F.C.
+Iran
+Iranian_Kurdistan
+Iraq
+Iraqi_Kurdistan
+Iraqi_National_Congress
+Ireland
+Ireland_national_rugby_union_team
+Irina_Korzhanenko
+Irina_Privalova
+Irina_Spîrlea
+Irish_Independent
+Irish_Republican_Army
+Irvine,_California
+Iryna_Yatchenko
+Isaac_Viciosa
+Isel_López
+Islam
+Islamabad
+Islamic_Museum
+Islamic_Republic_News_Agency
+Islamic_Republic_of_Iran
+Islamic_Salvation_Front
+Islamism
+Island_Beach_State_Park
+Isolde_Kostner
+Israel
+Israel_national_football_team
+Israel_national_under-21_football_team
+Israeli_Labor_Party
+Israelis
+Istanbul
+Isthmus_of_Tehuantepec
+Italy
+Italy_national_football_team
+Italy_national_rugby_union_team
+Itamar_Rabinovich
+Iulian_Filipescu
+Iva_Majoli
+Ivan_Francescato
+Ivan_Lendl
+Iván_García
+Iván_Rodríguez
+Iván_Zamorano
+J._Russell_(Essex_cricketer)
+JCPenney_Classic
+JK_Tallinna_Sadam
+Jaap_Stam
+Jacek_Dembiński
+Jack_Charlton
+Jack_Kemp
+Jack_Kevorkian
+Jack_Pierce_(athlete)
+Jackie_McNamara
+Jackie_Stewart
+Jacksonville,_Florida
+Jacksonville_Jaguars
+Jacky_Martens
+Jacob_Brumfield
+Jacques_Chirac
+Jacques_Kallis
+Jacques_Toubon
+Jacques_Villeneuve
+Jacqui_Cooper
+Jaffna
+Jaime_Oncins
+Jakarta
+Jakob_Hlasek
+Jalal_Talabani
+Jalalabad
+Jamaat-e-Islami
+Jamaica
+James_Baker
+James_Baldwin_(baseball)
+James_Brady
+James_Dalton_(rugby_player)
+James_Debbah
+James_Heath_(Boxer)
+James_Love
+James_Van_Allen
+James_Weaver_(racing_driver)
+Jamie_Baulch
+Jamie_Moyer
+Jamie_Spence
+Jan_Bos
+Jan_Ove_Pedersen
+Jan_Siemerink
+Jan_van_Eijden
+Jana_Kandarr
+Jana_Novotná
+Janakantha
+Janet_Reno
+Janette_Husárová
+Janez_Drnovšek
+Jani_Soininen
+Janne_Ojanen
+Jans_Koerts
+Jansher_Khan
+Jaora
+Japan
+Japan_Fed_Cup_team
+Japan_national_football_team
+Japie_Mulder
+Jared_Palmer
+Jared_Tomich
+Jari_Litmanen
+Jarmo_Kytölehto
+Jason_Belser
+Jason_Dickson
+Jason_Gillespie
+Jason_Little_(rugby_union)
+Jason_McAteer
+Jason_Rouser
+Jason_Stoltenberg
+Jason_Weaver_(jockey)
+Java_(programming_language)
+Javagal_Srinath
+Javier_Clemente
+Javier_Frana
+Javier_Sánchez
+Jay_Buhner
+Jean-Denis_Délétraz
+Jean-Louis_Debré
+Jean-Luc_Brassard
+Jean-Marc_Gounon
+Jean-Michel_Bayle
+Jean-Paul_van_Gastel
+Jean-Philippe_Fleurian
+Jean-Pierre_Cyprien
+Jean_Alesi
+Jean_Carlo_Witte
+Jean_Galfione
+Jean_Van_de_Velde
+Jearl_Miles_Clark
+Jed-Forest_RFC
+Jeddah
+Jeff_Bagwell
+Jeff_Bean
+Jeff_Parrett
+Jeff_Russell
+Jeff_Tarango
+Jeff_Williams_(athlete)
+Jeff_Wilson_(sportsman)
+Jeffrey_Lurie
+Jennifer_Capriati
+Jennifer_Flavin
+Jens_Fiedler_(cyclist)
+Jens_Todt
+Jeremy_Guscott
+Jeremy_Wotherspoon
+Jeroen_Blijlevens
+Jerry_Kelly
+Jersey
+Jerusalem
+Jesper_Parnevik
+Jesper_Skibby
+Jesse_Owens
+Jesus
+Jesús_Arellano
+Jesús_Tavárez
+Jews
+Jiang_Zemin
+Jiangsu
+Jiangxi
+Jill_Craybas
+Jim_Bolger
+Jim_Courier
+Jim_Edmonds
+Jim_Furyk
+Jim_Grabb
+Jim_Harbaugh
+Jim_Hines
+Jim_Leyritz
+Jim_Payne_(golfer)
+Jim_Telfer
+Jim_Thome
+Jimi_Hendrix
+Jimmy_Adams
+Jimmy_Key
+Jimmy_Thomson_(footballer)
+Jimmy_Vasser
+Jimy_Szymanski
+Jiří_Dopita
+Jiří_Džmura
+Jiří_Novák
+Joakim_Haeggman
+Joakim_Persson
+Joan_Llaneras
+Joannette_Kruger
+Joaquín_del_Olmo
+Jocelyn_Gourvennec
+Jodie_Foster
+Jody_Reed
+Joe-Max_Moore
+Joe_Carter
+Joe_Kneipp
+Joe_Miller_(footballer)
+Joe_Roff
+Joel_Stransky
+Joey_Hamilton
+Johan_Capiot
+Johan_Museeuw
+Johan_de_Kock
+Johann_Sebastian_Bach
+Johannesburg
+John_Boles_(baseball)
+John_C._Kornblum
+John_Collins_(footballer)
+John_Cook_(golfer)
+John_Crawley
+John_Fashanu
+John_Hagelin
+John_Hart_(rugby_coach)
+John_Hartson
+John_Hinckley,_Jr.
+John_Howard
+John_J._Sheehan
+John_Jaha
+John_Kerry
+John_Kocinski
+John_Langmore
+John_Lewis_Partnership
+John_Major
+John_Mark_Inienger
+John_Marzano
+John_Mayock
+John_McNamara_(baseball)
+John_Michael_Gorst
+John_Newcombe
+John_Regis_(athlete)
+John_Robinson_(footballer)
+John_Smiley
+John_Smoltz
+John_Stephenson_(cricketer,_born_1965)
+John_Talen
+John_Toshack
+John_Veldman
+John_White_(squash_player)
+John_Winslow_Bissell
+John_Y._Simon
+Johnny_Bench
+Johnny_Damon
+Johnny_Herbert
+Johnny_Oates
+Johns_Hopkins_Hospital
+Johnson_Controls
+Johor_Bahru
+Jon_Dahl_Tomasson
+Jon_Drummond
+Jonah_Lomu
+Jonas_Björkman
+Jonas_Savimbi
+Jonathan_Akpoborie
+Jonathan_Bachini
+Jonathan_Edwards_(athlete)
+Jonathan_Lomas
+Jonathan_Stark
+Jonathon_Power
+Jonty_Rhodes
+Joost_van_der_Westhuizen
+Jordan
+Jordan_Grand_Prix
+Jordan_River
+Jordi_Burillo
+Jordi_Cruyff
+Jorge_Cadete
+Jorge_Costa
+Jos_Verstappen
+Josef_Zieleniec
+Joseph_Bernardin
+Joseph_Keter
+Joseph_Nanven_Garba
+Joseph_Stalin
+Josh_Kronfeld
+José_Antonio_Escuredo
+José_Cóceres
+José_Emilio_Amavisca
+José_Hernández_(baseball)
+José_Herrera_(1990s_outfielder)
+José_Luis_Caminero
+José_Luís_Vidigal
+José_Maria_Cañizares
+José_María_Aznar
+José_María_Olazábal
+José_Miguel_Insulza
+José_Offerman
+José_Parra_(baseball)
+José_Ramos-Horta
+José_Rivero
+José_Rosado
+José_Taira
+José_Valentín
+João_Domingos_Pinto
+João_Havelange
+João_Vieira_Pinto
+Joël_Smets
+Juan_Antonio_Pizzi
+Juan_Curuchet
+Juan_Esnáider
+Juan_González_(baseball)
+Juan_Guzmán_(baseball)
+Juan_Perón
+Juan_Roque
+Juan_Somavía
+Judith_Arndt
+Judith_Wiesner
+Juha_Kankkunen
+Juha_Ylönen
+Julen_Lopetegui
+Jules_Maigret
+Julia_Carling
+Julia_Roberts
+Juliana_of_the_Netherlands
+Juliet_Cuthbert
+Julio_Dely_Valdés
+Julius_Nyerere
+Julián_Tavárez
+Jumet
+Junior_Murray
+Justin_Hobday
+Justin_Leonard
+Justin_Marshall
+Justin_Thompson
+Justin_Vaughan
+Juventus_F.C.
+Jyrki_Lumme
+Jyväskylä
+Ján_Krošlák
+Jörg_Heinrich
+Jürgen_Klinsmann
+Jürgen_Kohler
+Jürgen_Schult
+K.A.A._Gent
+K.F.C._Lommel_S.K.
+K.R.C._Genk
+K.R.C._Zuid-West-Vlaanderen
+K.S.C._Lokeren_Oost-Vlaanderen
+K._Sint-Truidense_V.V.
+KBC_Bank
+KIL_Toppfotball
+KK_Cibona
+KK_Crvena_zvezda
+KK_Partizan
+KK_Split
+KSE_100_Index
+KTM
+KV_Mechelen
+Ka_Wah_Bank
+Kabul
+Kader_Ferhaoui
+Kamiel_Maase
+Kampala
+Kandahar
+Kanpur
+Kansai_International_Airport
+Kansas
+Kansas_City,_Missouri
+Kansas_City_Chiefs
+Kansas_City_Royals
+Kaohsiung
+Kaolack
+Kapil_Dev
+Karachi
+Karachi_Stock_Exchange
+Karen_National_Union
+Karen_people
+Karim_Alami
+Karin_Janke
+Karin_Kschwendt
+Karina_Habšudová
+Karl-Heinz_Riedle
+Karl_Inderfurth
+Karl_Marx
+Karlsruher_SC
+Karol_Kučera
+Karsten_Bäron
+Kashmir
+Kassala
+Kassala_(state)
+Katarína_Studeníková
+Kate_Michelman
+Kate_Pace
+Katharina_Gutensohn
+Kathmandu
+Kathy_Rinaldi
+Katja_Seizinger
+Kawasaki_Heavy_Industries
+Kawasaki_Motors_Racing
+Kazakhstan
+Kazanlak
+Kazuyoshi_Funaki
+Kazuyoshi_Miura
+Keane_(company)
+Kecskemét
+Kehl
+Keirin
+Keith_Fletcher
+Keith_Wright_(footballer)
+Kelli_Kuehne
+Ken_Caminiti
+Ken_Green_(golfer)
+Ken_Griffey,_Jr.
+Ken_Hill_(baseball)
+Ken_Saro-Wiwa
+Kengo_Wa_Dondo
+Kenichi_Shimokawa
+Kennedy_Ochieng
+Kennedy_family
+Kennet_Andersson
+Kenneth_Benjamin
+Kenneth_Carlsen
+Kenneth_Clarke
+Kenneth_Eriksson
+Kenny_Cunningham
+Kenny_Dalglish
+Kenny_Harrison
+Kenny_Lofton
+Kenny_Rogers_(baseball)
+Kent_County_Cricket_Club
+Kentucky
+Kenya
+Kenya_national_cricket_team
+Kenya_national_football_team
+Kerwin_Bell
+Kevin_Brown_(right-handed_pitcher)
+Kevin_Foster_(baseball)
+Kevin_Keegan
+Kevin_Kim
+Kevin_Mitchell_(baseball)
+Kevin_Seitzer
+Kevin_Tapani
+KeyBank
+Khaled_Al_Zaher
+Khaleda_Zia
+Khalid_Boulami
+Khandwa
+Khartoum
+Khasavyurt
+Khmer_people
+Kia_Motors
+Kid_Brands
+Kiev
+Kigali
+Kilmarnock_F.C.
+Kim_Graham
+Kim_Yoon-man
+Kim_Young-sam
+Kimberley,_Northern_Cape
+Kimberly_Po
+Kimiko_Date_Krumm
+KinderCare_Learning_Centers
+Kindu
+King_Baudouin_Stadium
+Kingdom_of_England
+Kingdome
+Kingston_Technology
+Kinshasa
+Kirkcaldy
+Kirkland,_Washington
+Kisangani
+Kitty_Kiernan
+Kivu
+Klas_Eriksson
+Klaus_Kinkel
+Kleiner_Perkins_Caufield_&_Byers
+Knowsley_Road
+Kobus_Wiese
+Kofi_Annan
+Koha_Jonë
+Kol_Yisrael
+Kolkata
+Komerční_banka
+Kompas
+Konstantinos_Mitsotakis
+Konstantinos_Stephanopoulos
+Korea
+Korea_Republic_national_football_team
+Kosovo
+Kostas_Skandalidis
+Kragujevac
+Krasnoyarsk_Krai
+Krassimir_Balakov
+Kristian_Brenden
+Kristie_Boogert
+Kristina_Brandi
+Krzysztof_Piskuła
+Krzysztof_Warzycha
+Kuala_Lumpur
+Kubilay_Türkyilmaz
+Kumar_Dharmasena
+Kunar_River
+Kurdish_nationalism
+Kurdish_people
+Kurdistan
+Kurdistan_Democratic_Party
+Kurdistan_Workers'_Party
+Kurt_Abbott
+Kurt_Betschart
+Kurt_Schork
+Kuusamo
+Kuwait
+Kuwait_national_football_team
+KwaZulu-Natal
+Kyle_Abbott
+Kyran_Bracken
+Kyōko_Nagatsuka
+L-Orizzont
+LASK_Linz
+LG_Telecom
+LPGA_Championship
+LaMont_Smith
+La_Gazzetta_dello_Sport
+La_Jornada
+La_Nación
+La_Plagne
+La_Presse_(French_newspaper)
+La_Stampa
+La_création_du_monde
+Laban_Rotich
+Labor_Day
+Labour_Day
+Labour_Party_(Malta)
+Labour_Party_(UK)
+Lagos
+Lahore
+Lake_Maracaibo
+Lamberto_Dini
+Lamonts
+Lanarkshire
+Lancashire_County_Cricket_Club
+Lance_Armstrong
+Lance_Gibbs
+Land_Securities
+Landgraaf
+Landskrona
+Lansana_Conté
+Lantau_Peak
+Larisa_Neiland
+Larnaca
+Larry_Walker
+Lars_Bohinen
+Lars_Riedel
+Las_Cruces,_New_Mexico
+Las_Vegas,_Nevada
+Latin_America
+Latvia
+Latvia_national_football_team
+Laurence_Courtois
+Laurent-Désiré_Kabila
+Laurent_Blanc
+Laurent_Bénézech
+Laurent_Cabannes
+Laurent_Charvet
+Laurent_Gané
+Laurent_Jalabert
+Laurent_Ottoz
+Laurie_Brereton
+Laurie_Harper
+Lawrence_B._Lindsey
+Lawrence_of_Arabia_(film)
+Laxmi_Poruri
+Le_Canard_enchaîné
+Le_Figaro
+Le_Havre_AC
+Le_Jour
+Le_Matin_(France)
+Le_Monde
+Le_Temps
+League_of_Women_Voters
+Leah_Pells
+Leander_Paes
+Lebanon
+Lech_Poznań
+Lee's_Summit,_Missouri
+Lee_Bowyer
+Lee_Germon
+Lee_Sharpe
+Lee_Tinsley
+Lee_Wan_Wah
+Lee_Westwood
+Leeds
+Leeds_Rhinos
+Leeds_United_A.F.C.
+Lega_Nord
+Legal_Department
+Legia_Warszawa
+Lehman_Brothers
+Leicester
+Leicester_City_F.C.
+Leicester_Tigers
+Leicestershire_County_Cricket_Club
+Leipzig/Halle_Airport
+Lennart_Meri
+Lenzing_AG
+Leonid_Kuchma
+Leopold_III_of_Belgium
+Les_Ferdinand
+Les_Échos_(France)
+Letitia_Vriesde
+Levothyroxine
+Lewis_Gunn
+Leyton_Orient_F.C.
+Lhasa
+Li_Peng
+Liam_Botham
+Liam_Daish
+Liam_Fox
+Liam_Gallagher
+Liam_Neeson
+Liaoning
+Liberal_Democratic_Party_(Japan)
+Liberal_Party_(UK)
+Liberal_Party_of_Australia
+Liberation_Tigers_of_Tamil_Eelam
+Liberia
+Liberia_national_football_team
+Libya
+Lidl
+Liechtenstein
+Liechtenstein_national_football_team
+Lien_Chan
+Lierse_S.K.
+Lihir_Island
+Likud
+Lilian_Thuram
+Lille_OSC
+Lillehammer
+Lima
+Limerick
+Limited_company
+Lincoln,_Nebraska
+Lincoln_City_F.C.
+Linda_Kisabaka
+Linda_Wild
+Lindsay_Davenport
+Lindy_Remigino
+Linfield_F.C.
+Linford_Christie
+Linz
+Lisa_Raymond
+Lisbeth_Stuer-Lauridsen
+Lisbon
+Lithuania
+Lithuania_national_football_team
+Lithuania_national_under-21_football_team
+Little_Rock,_Arkansas
+Liverpool_F.C.
+Livingston_F.C.
+Liviu_Ciobotariu
+Lixion_Avila
+Liège
+Ljubljana
+Llanelli_RFC
+Lloyd_Axworthy
+Lloyd_Bentsen
+Lockerbie
+Loftus_Versfeld_Stadium
+Lombardi_Award
+Lomé
+London
+London_Heathrow_Airport
+London_Interbank_Offered_Rate
+London_Irish
+London_Metal_Exchange
+London_Stansted_Airport
+London_Stock_Exchange
+London_Wasps
+Long_Island
+Longchamp_Racecourse
+Longyearbyen
+Lonmin
+Loren_Roberts
+Lorenzo_Amoruso
+Lori_McNeil
+Los_Angeles
+Los_Angeles_Angels_of_Anaheim
+Los_Angeles_Clippers
+Los_Angeles_Dodgers
+Los_Angeles_Kings
+Los_Angeles_Lakers
+Lou_Gehrig
+Louis_Farrakhan
+Louis_Freeh
+Louise_Currey
+Loyola_de_Palacio
+Luanda
+Luc_Nilis
+Luca_Cadalora
+Lucas_Moreira_Neves
+Lucy_Tyler-Sharman
+Ludmila_Engquist
+Ludmila_Richterová
+Ludovic_Giuly
+Lufthansa
+Luis_Enrique_Martínez_García
+Luis_García_Postigo
+Luis_Milla
+Luis_Oliveira
+Luis_Polonia
+Luke_Jensen
+Luke_Kipkosgei
+Lukoil
+Lusaka_Protocol
+Luton_Town_F.C.
+Luxembourg
+Luxembourg_(city)
+Luís_Figo
+Lynda_Chalker,_Baroness_Chalker_of_Wallasey
+Lyngby_Boldklub
+M._Karunanidhi
+MBC_Dynamo_Moscow
+MCI_Communications
+MFK_Dubnica
+MFK_Košice
+MFK_Petržalka
+MMC_Norilsk_Nickel
+MSV_Duisburg
+MTK_Hungária_FC
+MYPA
+Maariv_(newspaper)
+Maarten_den_Bakker
+Maccabi_Haifa_F.C.
+Maccabi_Herzliya_F.C.
+Maccabi_Petah_Tikva_F.C.
+Maccabi_Tel_Aviv_B.C.
+Maccabi_Tel_Aviv_F.C.
+Macedonia_national_football_team
+Madagascar
+Madame_Bovary
+Madeleine_Albright
+Madhumalla
+Madrid
+Mae_Sot
+Mafikeng
+Magdalena_Grzybowska
+Magdalena_Maleeva
+Magna_International
+Magnus_Gustafsson
+Magnus_Larsson
+Mahala
+Mahmoud_Abbas
+Major_League_Baseball
+Major_League_Baseball_Most_Valuable_Player_Award
+Maksim_Tarasov
+Malawi
+Malawi_national_football_team
+Malaysia
+Malaysia_Open_(badminton)
+Malcolm_Marshall
+Malcolm_Rifkind
+Mali
+MaliVai_Washington
+Malta
+Malta_Freeport
+Malta_national_football_team
+Managua
+Manama
+Manchester
+Manchester_City_F.C.
+Manchester_United_F.C.
+Mandsaur
+Manfred_Kanther
+Manfred_Schwabl
+Mangosuthu_Buthelezi
+Manhattan
+Manhattan_Beach,_California
+Maniema
+Manly-Warringah_Sea_Eagles
+Mansfield_Town_F.C.
+Manuel_Medina_(boxer)
+Mao_Zedong
+Maputo
+Maracaibo
+Marc-Vivien_Foé
+Marc_Blume
+Marc_Degryse
+Marc_Dutroux
+Marc_Hottiger
+Marc_Keller
+Marc_Newfield
+Marc_Overmars
+Marc_Rosset
+Marc_Wilkins_(baseball)
+Marc_Wilmots
+Marcel_Desailly
+Marcello_Cuttitta
+Marcelo_Filippini
+Marcelo_Otero
+Marcelo_Ríos
+Marcelo_Salas
+Marcelo_Silva_Ramos
+Marcin_Mięciel
+Marco_Bode
+Marco_Delvecchio
+Marco_Lietti
+Marco_Pascolo
+Marco_Vaccari
+Marco_Villa
+Marcos_Ondruska
+Marcus_Gayle
+Marcus_Grönholm
+Marek_Citko
+Margaret_Crowley_(athlete)
+Margaret_Thatcher
+Maria_Mutola
+Mariaan_de_Swardt
+Marianne_Timmer
+Marianne_Werdel
+Mariano_Bombarda
+Mariano_Duncan
+Mariano_Juan
+Mariano_Rivera
+Maribor
+Marie-José_Pérec
+Marie_Lindgren
+Marina_Trandenkova
+Mario_Basler
+Mario_Silva
+Mario_Stanić
+Marion_Clignet
+Marius_Hurter
+Mark_Acre
+Mark_Andrews_(rugby_player)
+Mark_Brooks_(golfer)
+Mark_Butcher
+Mark_Cairns_(squash_player)
+Mark_Calcavecchia
+Mark_Chaloner
+Mark_Crear
+Mark_Davis_(golfer)
+Mark_Dekker
+Mark_Ealham
+Mark_Gleeson_(journalist)
+Mark_Hughes
+Mark_Hutton
+Mark_Ilott
+Mark_Kennedy_(footballer)
+Mark_Knowles
+Mark_Lawrence_Wolf
+Mark_McGwire
+Mark_McNulty
+Mark_Mouland
+Mark_Pembridge
+Mark_Petkovsek
+Mark_Philippoussis
+Mark_Prescott
+Mark_Richardson_(athlete)
+Mark_Roe
+Mark_Rutherford_(footballer)
+Mark_Spitz
+Mark_Taylor_(cricketer)
+Mark_Thompson_(baseball)
+Mark_Waugh
+Mark_Woodforde
+Markham,_Ontario
+Marko_Koers
+Markus_Babbel
+Markus_Feldhoff
+Markus_Schopp
+Marlene_Thomsen
+Maronite_Church
+Marseille
+Marshall_Faulk
+Martela
+Martin_Brundle
+Martin_Damm
+Martin_Driller
+Martin_Heath
+Martin_Max
+Martin_McCague
+Martina_Ertl-Renz
+Martina_Hingis
+Marty_Cordova
+Marty_Nothstein
+Marubeni
+Marvin_Benard
+Marvin_Harrison
+Marxism
+Mary_Joe_Fernandez
+Mary_Onyali-Omagbemi
+Mary_Pierce
+Marylebone_Cricket_Club
+Masakiyo_Maezono
+Masami_Ihara
+Masayoshi_Takemura
+Masayuki_Okano
+Massachusetts
+Massimo_Cuttitta
+Massoud_Barzani
+Massoud_Rajavi
+Matagalpa
+Mate_Granić
+Mathias_Jack
+Mathias_Ntawulikura
+Mats_Lanner
+Mats_Wilander
+Matt_Burke
+Matt_Lawton
+Matt_Stairs
+Matthew_Le_Tissier
+Matthew_Maynard
+Matthew_Pinsent
+Matthew_Windows
+Matthias_Hagner
+Matthias_Sammer
+Matuzići
+Maurice_Baril
+Maurice_Graef
+Mauricio_Hadad
+Mauritania
+Mauritania_national_football_team
+Mauritius
+Mauritius_national_football_team
+Maurizio_Fondriest
+Mauro_Silva
+Maurício_Gugelmin
+Max_Delbrück
+Max_Sciandri
+Max_van_Heeswijk
+Małgorzata_Rydz
+McLaren
+Mecca
+Medellín
+Media_Indonesia
+Mediaset
+Medicare_(United_States)
+Mediterranean_Sea
+Megawati_Sukarnoputri
+Meghann_Shaughnessy
+Mehmet_Scholl
+Melbourne
+Melbourne_Cricket_Ground
+Melbourne_Football_Club
+Melrose_RFC
+Meluawati
+Melvin_Nieves
+Memphis_Grizzlies
+Menlo_Park,_California
+Mercedes-Benz
+Meredith_McGrath
+Merlene_Ottey
+Merrill_Lynch
+Metro_AG
+Metro_Manila
+Metropolitan_Fiber_Systems
+Metropolitan_Museum_of_Art
+Mette_Bergmann
+Mexican_Army
+Mexico
+Mexico_City
+Mexico_national_football_team
+Mhow
+Mia_Audina
+Miami
+Miami_Dolphins
+Miami_Heat
+Michael_Andersson
+Michael_Andretti
+Michael_Bevan
+Michael_Branch
+Michael_Campbell
+Michael_Chang
+Michael_Collins_(Irish_leader)
+Michael_Collins_(film)
+Michael_Di_Venuto
+Michael_Doohan
+Michael_Green_(sprinter)
+Michael_Heseltine
+Michael_Hübner
+Michael_J._Astrue
+Michael_Jeffrey
+Michael_Johnson_(athlete)
+Michael_Jones_(rugby_union)
+Michael_Jonzon
+Michael_Joyce_(tennis)
+Michael_Konsel
+Michael_Lynagh
+Michael_Marsh_(athlete)
+Michael_Möllenbeck
+Michael_Reiziger
+Michael_Schumacher
+Michael_Slater
+Michael_Stich
+Michael_Tebbutt
+Michael_Tucker_(baseball)
+Michael_Zorc
+Michaela_Dorfmeister
+Michel_Hansenne
+Michele_Bartoli
+Michelle_Freeman
+Michigan
+Michigan_City,_Indiana
+Michigan_National_Bank
+Michoacán
+Mick_McCarthy
+Mickey_Cochrane
+Mickey_Kantor
+Mickey_Rivers
+Microsoft
+Microsoft_Windows
+Middle_East
+Middle_East_Economic_Survey
+Middle_Kingdom_of_Egypt
+Middlesbrough_F.C.
+Middlesex_County_Cricket_Club
+Midwestern_United_States
+Miguel_Ángel_Jiménez
+Miguel_Ángel_Martín_(golfer)
+Miguel_Ángel_Nadal
+Mihai_Tararache
+Mika_Häkkinen
+Mika_Salo
+Mikael_Tillström
+Mike_Atherton
+Mike_Conley,_Sr.
+Mike_Devereaux
+Mike_Fetters
+Mike_Gatting
+Mike_Harwood
+Mike_Hulbert
+Mike_McCurry_(press_secretary)
+Mike_Mordecai
+Mike_Oquist
+Mike_Piazza
+Mike_Stanley
+Mike_Sullivan_(golfer)
+Mike_Tyson
+Mike_Watkinson
+Mike_Williams_(baseball)
+Mikhail_Gorbachev
+Mikhail_Kutuzov
+Mil_Mi-17
+Miladin_Bečanović
+Milan
+Milan_Kučan
+Miles_Tunnicliff
+Milinko_Pantić
+Million_Man_March
+Millwall_F.C.
+Miltiadis_Evert
+Milwaukee
+Milwaukee_Brewers
+Milwaukee_Bucks
+Minardi
+Mindanao
+Ming_Pao
+Minister_for_Foreign_Affairs_(Australia)
+Minister_of_the_Interior_(France)
+Ministry_of_Finance_(Chile)
+Ministry_of_Finance_(Japan)
+Ministry_of_Foreign_Affairs_(Israel)
+Ministry_of_Foreign_Affairs_(Poland)
+Ministry_of_Foreign_Affairs_(Turkey)
+Ministry_of_Foreign_Affairs_of_the_People's_Republic_of_China
+Minneapolis
+Minnesota
+Minnesota_Timberwolves
+Minnesota_Twins
+Minnesota_Vikings
+Minnetonka,_Minnesota
+Minsk
+Miriam_Oremans
+Miriam_Vogt
+Miron_Cozma
+Miroslav_Votava
+Miss_Universe
+Missionaries_of_Charity
+Mississippi_River
+Missouri
+Mitsubishi_Lancer
+Mitsubishi_Motors
+Mitsui
+Mladá_fronta_DNES
+Mo_Vaughn
+Mobil
+Mobutu_Sese_Seko
+Moengo
+Mohali
+Mohammad_Akram
+Mohammad_Azharuddin
+Moin_Khan
+Molde_FK
+Moldova
+Moldova_national_football_team
+Mona_Eltahawy
+Monaco
+Monarcas_Morelia
+Mongolia
+Monica_Seles
+Monrovia
+Mons_Ivar_Mjelde
+Mont-sur-Marchienne
+Montana
+Monte_Carlo
+Montenegrins
+Montenegro
+Monterey,_California
+Montgomery,_Alabama
+Montpelier,_Vermont
+Montpellier_HSC
+Montreal
+Montreal_Canadiens
+Montreal_Expos
+Montrose_F.C.
+Montserrat
+Morelia
+Morgan_Stanley
+Morinville,_Alberta
+Moro_National_Liberation_Front
+Morocco
+Moscow
+Moscow_Kremlin
+Mosenergo
+Moses_Kiptanui
+Moss_FK
+Moss_Landing_Marine_Laboratories
+Mostostal
+Mother_Teresa
+Motherwell_F.C.
+Motohiro_Yamaguchi
+Mountain_View,_California
+Moustapha_Niasse
+Movement_of_Socialist_Democrats
+Movladi_Udugov
+Mozambique
+Mozambique_national_football_team
+Mr._Clean
+Muammar_al-Gaddafi
+Muhammad
+Mumbai
+Munich
+Munich_Airport
+Munich_Re
+Murat_Yakin
+Murphy_Jensen
+Murrayfield_Stadium
+Mushtaq_Ahmed
+Muslim_Commercial_Bank
+Mustapha_Hadji
+Muttahida_Qaumi_Movement
+Muttiah_Muralitharan
+Muzaffarabad
+My_Turn_(memoir)
+Márcio_Roberto_dos_Santos
+Mário_Covas
+Mário_Jardel
+Médecins_Sans_Frontières
+MŠK_Rimavská_Sobota
+MŠK_Žilina
+N.E.C._(football_club)
+NAC_Breda
+NARAL_Pro-Choice_America
+NASDAQ
+NATO
+NATO_intervention_in_Bosnia
+NBC
+NK_Olimpija_Ljubljana_(1911)
+NK_Rudar_Velenje
+NK_Varaždin
+NSDAP/AO
+NTV_(Russia)
+Nabih_Berri
+Nablus
+Nagorno-Karabakh_Republic
+Nagoum_Yamassoum
+Nagoya
+Nagyatád
+Nail_Beširović
+Naina_Yeltsina
+Nairobi
+Namibia
+Namibia_national_football_team
+Nana_Miyagi
+Nanabhoy_Palkhivala
+Nancy_Reagan
+Nandrin
+Naoki_Soma
+Naoko_Sawamatsu
+Naoto_Kan
+Naples
+Napoleon_I
+Napoleon_III
+Naqoura
+Narciso_dos_Santos
+Naseem_Hamed
+Nasrallah_Boutros_Sfeir
+Nasser_Hussain
+Natalya_Pomoshchnikova-Voronova
+Natalya_Sadova
+Natalya_Shikolenko
+Natalya_Torshina-Alimzhanova
+Natasha_Zvereva
+Nate_Miller
+Nathalie_Dechy
+Nathalie_Lancien
+Nathalie_Tauziat
+Nathan_Astle
+Nation_of_Islam
+National_Air_Traffic_Services
+National_Alliance_(Italy)
+National_Assembly_of_Angola
+National_Bank_(Bangladesh)
+National_Bank_of_Poland
+National_Basketball_Association
+National_Collegiate_Athletic_Association
+National_Congress_of_Brazil
+National_Democratic_Alliance_(Sudan)
+National_Football_League
+National_Guard_of_the_United_States
+National_Hockey_League
+National_Human_Rights_Commission_(Mexico)
+National_Hurricane_Center
+National_Iraqi_News_Agency
+National_Islamic_Front
+National_League
+National_League_Central
+National_League_East
+National_League_West
+National_League_for_Democracy
+National_Liberation_Front_of_Corsica
+National_Olympic_Committee
+National_Park_Service
+National_Party_(South_Africa)
+National_Rifle_Association
+National_Tennis_Centre_(United_Kingdom)
+National_Transportation_Safety_Board
+National_University_of_Singapore
+National_Weather_Service
+National_stadium
+NationsBank
+Native_Americans_in_the_United_States
+Natural_Law_Party_(United_States)
+Nature_(journal)
+Navjot_Singh_Sidhu
+Nawaz_Sharif
+Nayan_Mongia
+Nazi_Germany
+Nazi_Party
+Nazism
+Neal_Lancaster
+Neale_Fraser
+Neath_RFC
+Nebraska
+Nebraska_Cornhuskers_football
+Necmettin_Erbakan
+Neemuch
+Neil_Back
+Neil_Fairbrother
+Neil_Hodgson
+Neil_Jordan
+Neil_Williams_(cricketer)
+Nelson,_New_Zealand
+Nelson_A._Miles
+Nelson_Mandela
+Nelė_Žilinskienė
+Neo-Nazism
+Nepal
+Nepali_Congress
+Netherlands
+Netherlands_Fed_Cup_team
+Netherlands_national_football_team
+Neuchâtel
+Neuchâtel_Xamax
+Nevada
+Nevil_Dede
+Nevill_Ground
+Neville_Godwin
+NewYork–Presbyterian_Hospital
+New_Aspiration_Party
+New_Caledonia
+New_Delhi
+New_Democracy_(Greece)
+New_England
+New_England_Patriots
+New_Hampshire,_Ohio
+New_Haven,_Connecticut
+New_Jersey
+New_Jersey_Devils
+New_Jersey_Nets
+New_Mexico
+New_Mexico_Activities_Association
+New_Orleans_Hornets
+New_Orleans_Saints
+New_Party_Sakigake
+New_Scientist
+New_South_Wales_rugby_league_team
+New_Territories
+New_York
+New_York_City
+New_York_City_Department_of_Transportation
+New_York_Cotton_Exchange
+New_York_Giants
+New_York_Islanders
+New_York_Jets
+New_York_Knicks
+New_York_Mercantile_Exchange
+New_York_Mets
+New_York_Post
+New_York_Rangers
+New_York_Stock_Exchange
+New_York_Yankees
+New_Zealand
+New_Zealand_Barbarians
+New_Zealand_First
+New_Zealand_Labour_Party
+New_Zealand_National_Party
+New_Zealand_Press_Association
+New_Zealand_Warriors
+New_Zealand_dollar
+New_Zealand_national_cricket_team
+New_Zealand_national_rugby_union_team
+New_Zealanders
+New_media
+Newark,_New_Jersey
+Newbridge_RFC
+Newbury,_Berkshire
+Newcastle_Knights
+Newcastle_United_F.C.
+Newcastle_upon_Tyne
+Newcrest_Mining
+Newfoundland_(island)
+Newmarket,_Suffolk
+Newmont_Mining_Corporation
+Newport_Gwent_Dragons
+Newport_RFC
+News_Corporation
+Newsnight
+Newt_Gingrich
+Niall_Quinn
+Nicaragua
+Nicaraguans
+Nice
+Nick_Faldo
+Nick_Farr-Jones
+Nick_Knight
+Nick_Nieland
+Nick_Popplewell
+Nick_Price
+Nicklas_Kulti
+Nicky_Boje
+Niclas_Fasth
+Nico-Jan_Hoogma
+Nico_Motchebon
+Nico_van_Kerckhoven
+Nicola_Mazzucato
+Nicolas_Ouédec
+Nicole_Arendt
+Nicole_Bradtke
+Nicole_Brown_Simpson
+Nicoleta_Grasu
+Nicolás_Lapentti
+Nicolás_Pereira
+Nicosia
+Nigel_Mobbs
+Nigel_Walker
+Niger
+Nigeria
+Nihon_Keizai_Shimbun
+Nikkei_225
+Nir_Sivilia
+Nixon_McLean
+Nobel_Peace_Prize
+Nobel_Prize
+Nobina_Sverige
+Noel_Gallagher
+Noel_Whelan
+Nolan_Henke
+Nolan_Ryan
+Noranda_(mining_company)
+Norberto_Téllez
+Nordegg,_Alberta
+Noriaki_Kasai
+Norifumi_Abe
+Norio_Omura
+Noriyuki_Haga
+Norm_Charlton
+Norm_Hewitt
+Norodom_Ranariddh
+Norodom_Sihanouk
+North_America
+North_Battleford
+North_Caucasus
+North_Dakota
+North_Island
+North_Kivu
+North_Korea
+North_Melbourne_Football_Club
+North_Queensland_Cowboys
+North_Shore_City
+North_Sydney_Bears
+North_West_(South_African_province)
+North_Yemen
+Northampton
+Northampton_Saints
+Northampton_Town_F.C.
+Northamptonshire_County_Cricket_Club
+Northern_Cape
+Northern_Ireland
+Northern_Ireland_national_football_team
+Northerns_cricket_team
+Norway
+Norwest
+Norwich_City_F.C.
+Notting_Hill_Carnival
+Nottingham
+Nottingham_Forest_F.C.
+Nottinghamshire_County_Cricket_Club
+Notts_County_F.C.
+Nouakchott
+Noureddine_Morceli
+Novye_Atagi
+Numico
+Nunthorpe_Stakes
+Nuremberg
+Nuremberg_Airport
+O._J._Simpson
+OFK_Beograd
+OGC_Nice
+OKD
+OKS_1945_Olsztyn
+OPEC
+OSCE_Minsk_Group
+O_Globo
+Oakland,_California
+Oakland,_New_Jersey
+Oakland_Athletics
+Oakland_Raiders
+Oasis_(band)
+Oaxaca
+Oceano_da_Cruz
+Odra_Wodzisław
+Oerlikon_(Zürich)
+Office_of_Fair_Trading
+Ogün_Temizkanoğlu
+Ohio
+Ohio_State_Buckeyes_football
+Okinawa_Prefecture
+Oklahoma
+Oklahoma_Panhandle
+Oklahoma_State_University–Stillwater
+Okocim
+Oksana_Grishina
+Olabisi_Afolabi
+Olaf_Ludwig
+Olaf_Thon
+Old_City_(Jerusalem)
+Old_Testament
+Old_Trafford
+Old_Trafford_Cricket_Ground
+Oldham_Athletic_A.F.C.
+Oldham_Roughyeds
+Ole_Einar_Bjørndalen
+Ole_Gunnar_Solskjær
+Olga_Chernyavskaya
+Olga_Melnik
+Olga_Slyusareva
+Olin_Browne
+Oliver_Bierhoff
+Oliver_Kahn
+Oliver_McCall
+Oliver_Reck
+Olivier_Grouillard
+Olivier_Panis
+Olo_Brown
+Olympiacos_F.C.
+Olympiad
+Olympic_Airlines
+Olympic_sports
+Olympique_Khouribga
+Olympique_Lyonnais
+Olympique_de_Marseille
+Omaha,_Nebraska
+Oman
+Omar_Bongo
+Omar_Karami
+On_the_Waterfront
+Ong_Ewe_Hock
+Opel
+Operation_Provide_Comfort
+Oppenheimer_Holdings
+Oradea_International_Airport
+Oran
+Orel_Hershiser
+Organisation_of_the_Islamic_Conference
+Orhan_Çıkırıkçı
+Orlando_Magic
+Orlando_Miller
+Orlando_Pace
+Orlando_Pirates_FC
+Orrell_R.U.F.C.
+Orvieto
+Osaka
+Osaka_International_Airport
+Oscar_Luigi_Scalfaro
+Oslo
+Oslobođenje
+Osmond_Ezinwa
+Otis_Nixon
+Ottawa
+Ottawa_67's
+Ottawa_Senators
+Ottoman_Empire
+Ove_Sellberg
+Ovidiu_Stângă
+Owen_Finegan
+Oxford
+Oxford_United_F.C.
+Ozzie_Guillén
+Oğuz_Çetin
+P._W._Botha
+PBC_CSKA_Moscow
+PFC_CSKA_Moscow
+PFC_Levski_Sofia
+PFC_Nyva_Vinnytsia
+PGA_European_Tour
+PGA_Tour
+PMC-Sierra
+PSINet
+PSV_Eindhoven
+Pablo_Sánchez
+Pacific_Division_(NBA)
+Pacific_Division_(NHL)
+Pacific_Exchange
+Pacific_Rim
+Pacific_coast
+Paine_Webber
+Pakistan
+Pakistan_Muslim_League_(N)
+Pakistan_national_cricket_team
+Palestine
+Palestinian_Legislative_Council
+Palestinian_National_Authority
+Palestinian_territories
+Palk_Strait
+Pallacanestro_Treviso
+Pam_Shriver
+Pamir_Mountains
+Pamplona
+Pan-Turkism
+Pan_Am_Flight_103
+Pan_American_World_Airways
+Panama
+Panama_City
+Panama_national_football_team
+Panathinaikos_BC
+Panathinaikos_F.C.
+Panhellenic_Socialist_Movement
+Panionios_B.C.
+Paola_Suárez
+Paolo_Di_Canio
+Paolo_Negro
+Paolo_Vaccari
+Papendrecht
+Papua_New_Guinea
+Paraguay
+Paralympic_Games
+Paramaribo
+Paramount,_California
+Paramount_Pictures
+Paraná_Clube
+Parc_des_Princes
+Paris
+Paris,_Arkansas
+Paris-Charles_de_Gaulle_Airport
+Paris_Saint-Germain_F.C.
+Paris_Saint_Germain_(rugby_league_team)
+Park_Sung-hee
+Parma_F.C.
+Parramatta_Eels
+Partick_Thistle_F.C.
+Partition_of_India
+Party_of_Democratic_Action
+Party_of_Democratic_Socialism_(Germany)
+Pará
+Pascal_Renier
+Pascal_Zuberbühler
+Pat_Cash
+Pat_Hentgen
+Pat_Howard
+Pat_Hurst
+Pat_Manson
+Pat_McGinlay
+Pat_Symcox
+Patrice_Loko
+Patricia_Girard
+Patrick_Colleter
+Patrick_Kluivert
+Patrick_Rafter
+Patrick_Sang
+Patrick_Seale
+Patrick_Stevens
+Patrick_Sylvestre
+Patrick_Vieira
+Patrik_Andersson
+Patrik_Sjöberg
+Patriotic_Salvation_Movement
+Patriotic_Union_of_Kurdistan
+Patsy_Kensit
+Pattaya
+Paul_Adams_(cricketer)
+Paul_Affleck
+Paul_Belmondo
+Paul_Broadhurst
+Paul_Curry_(golfer)
+Paul_Eales
+Paul_Gascoigne
+Paul_Goydos
+Paul_Haarhuis
+Paul_Ince
+Paul_Johnson_(cricketer)
+Paul_Johnson_(squash_player)
+Paul_Justin
+Paul_Keating
+Paul_Koech
+Paul_Lawrie
+Paul_McGinley
+Paul_McGrath_(footballer)
+Paul_Newlove
+Paul_Peschisolido
+Paul_Prichard
+Paul_Reiffel
+Paul_Sorrento
+Paul_Stankowski
+Paul_Strang
+Paul_Tergat
+Paul_Tracy
+Paul_Van_Himst
+Paul_Warhurst
+Paul_Wright_(footballer)
+Paula_Radcliffe
+Pauline_Davis-Thompson
+Pauline_Konga
+Paulinho_Santos
+Paulo_Alves
+Paulo_Bento
+Paulo_Roberto_do_Carmo
+Paulo_Sousa
+Paulo_Sérgio_Silvestre_do_Nascimento
+Pavel_Buran
+Pavel_Bure
+Pavel_Muslimov
+Pavel_Nedvěd
+Pavel_Polomský
+Payne_Stewart
+Peace_River_(Canada)
+Pedro_Diniz
+Pedro_I_of_Brazil
+Pedro_Lamy
+Pedro_Linhart
+Pedro_Martínez
+Peer_Gynt
+Pemberton,_British_Columbia
+Pembroke,_Massachusetts
+Pembroke_Pines,_Florida
+Pendleton,_Oregon
+Pennsylvania
+Penrith_Panthers
+People's_Party_(Spain)
+People's_Party_–_Movement_for_a_Democratic_Slovakia
+People's_Republic_of_China
+People_of_the_United_States
+Per_Nyman
+Perm
+Pernilla_Wiberg
+Peronism
+Persian_Gulf
+Perth,_Western_Australia
+Peru
+Perugia_Calcio
+Pesalai
+Pete_Harnisch
+Pete_Incaviglia
+Pete_Sampras
+Pete_Wilson
+Peter_Atherton
+Peter_Baker_(golfer)
+Peter_Beardsley
+Peter_Blank
+Peter_Dubovský
+Peter_Hedblom
+Peter_Johansson_(motorcyclist)
+Peter_Kox
+Peter_Martin_(cricketer)
+Peter_Mitchell_(golfer)
+Peter_Munk
+Peter_Nicol
+Peter_Schöttel
+Peter_Such
+Peter_Tramacchi
+Peter_van_Vossen
+Peterborough_United_F.C.
+Petersburg,_Virginia
+Petr_Gabriel
+Petr_Korda
+Petra_Behle
+Petra_Langrová
+Petra_News_Agency
+Petre_Roman
+Phil_Babb
+Phil_Gray
+Phil_Mickelson
+Phil_Regan_(baseball)
+Phil_Simmons
+Philadelphia
+Philadelphia_76ers
+Philadelphia_Eagles
+Philadelphia_Flyers
+Philadelphia_Phillies
+Philip_Morris_International
+Philip_Walton
+Philippe_Ermenault
+Philippe_Sella
+Philippe_Séguin
+Philippine_Basketball_Association
+Philippines
+Phillip_Cocu
+Phillip_DeFreitas
+Phillip_Price
+Phnom_Penh
+Phoenix,_Arizona
+Phoenix_Coyotes
+Phoenix_Suns
+Phylis_Smith
+Piacenza_Calcio
+Picabo_Street
+Pierluigi_Casiraghi
+Pierre-Henri_Raphanel
+Pierre_Buyoya
+Pierre_Fulke
+Pierre_Laigle
+Pierre_Lueders
+Pierre_van_Hooijdonk
+Pilot_Pen_Tennis
+Piper_Jaffray
+Pirelli
+Pittsburgh
+Pittsburgh_Penguins
+Pittsburgh_Pirates
+Pittsburgh_Steelers
+Place_Beauvau
+Plymouth,_Montserrat
+Plymouth_Argyle_F.C.
+Podujevo
+Pohang_Steelers
+Pol_Pot
+Poland
+Poland_national_football_team
+Poles
+Polish_Press_Agency
+Polonia_Warszawa
+Pontefract
+Pontiac,_Michigan
+Pontypridd
+Pontypridd_RFC
+Pope_John_Paul_II
+Popular_Revolutionary_Army
+Porsche
+Port_Arthur,_Tasmania
+Port_Louis
+Port_Vale_F.C.
+Portadown_F.C.
+Portland,_Oregon
+Portland_Trail_Blazers
+Portsmouth
+Portsmouth_F.C.
+Portugal
+Portugal_national_football_team
+Portuguese_Empire
+Portuguese_Liga
+Post-Communism
+Post-Soviet_states
+Potsdam
+Poul-Erik_Høyer_Larsen
+Prague
+Prague_Stock_Exchange
+Prakash_Chandra_Lohani
+Predrag_Mijatović
+Premier_League
+Prensa_Latina
+Press_Trust_of_India
+Preston
+Preston_North_End_F.C.
+Prestwick
+Pretoria
+Primož_Peterka
+Prince_Eugene_of_Savoy
+Prince_Rupert,_British_Columbia
+Prince_William_Sound
+Princeton_N._Lyman
+Pro_Bowl
+Pro_Football_Hall_of_Fame
+Professional_Squash_Association
+Progressive_Field
+Province_of_Mantua
+Provisional_Irish_Republican_Army
+Prudential_plc
+Public_Transport_Corporation
+Public_utilities_commission
+Pudong
+Puebla_F.C.
+Puerto_Rico
+Puertollano
+Punjab,_Pakistan
+Pyramiden
+Pádraig_Harrington
+Pável_Pardo
+Qasr-e_Shirin
+Qian_Qichen
+Qing_Dynasty
+Queen's_Park,_Chesterfield
+Queen_of_the_South_F.C.
+Queens_Park_Rangers_F.C.
+Queensland
+Quentin_Coryatt
+Quetta
+Quintana_Roo
+Quito
+Qur'an
+R.E._Mouscron
+R.S.C._Anderlecht
+R.W.D._Molenbeek
+R._Charleroi_S.C.
+R._Nicholas_Burns
+RAI
+RAO_UES
+RCD_Espanyol
+RC_Lens
+RC_Strasbourg
+RKC_Waalwijk
+RMS_Titanic
+RSA_Security
+Rabat
+Rabobank
+Rabobank_(cycling_team)
+Racing_Club_de_Avellaneda
+Racing_de_Santander
+Radek_Bonk
+Radical_Cause
+Radio_Kabul
+Radio_New_Zealand
+Radiotelevision_of_Bosnia-Herzegovina
+Radka_Zrubáková
+Rafael_Alkorta
+Rafael_Palmeiro
+Rafic_Hariri
+Rahul_Dravid
+Rainier_III,_Prince_of_Monaco
+Raith_Rovers_F.C.
+Raja_Casablanca
+Raków_Częstochowa
+Ralf_Kelleners
+Rama
+Ramallah
+Rambo_(film_series)
+Rameswaram
+Ramla
+Ramon_Vega
+Ramón_Delgado
+Ramón_Ramírez
+Randall_Cunningham
+Randy_Johnson
+Randy_Jones_(bobsleigh)
+Randy_Velarde
+Rangers_F.C.
+Ranji_Trophy
+Ransart,_Belgium
+Raphaël_Wicky
+Ras_Tanura
+Rashid_Latif
+Rashid_Sidek
+Ratlam
+Ravenna_Calcio
+Ravi_River
+Ravindra_Pushpakumara
+Ray_Buchanan
+Ray_Durham
+Ray_Houghton
+Ray_Knight
+Ray_Lankford
+Raymond_Atteveld
+Raymond_James
+Raymond_Russell
+Rayo_Vallecano
+Raúl_González
+Reading_F.C.
+Reading_R.F.C.
+Real_Betis
+Real_Madrid_C.F.
+Real_Oviedo
+Real_Sociedad
+Real_Valladolid
+Real_Zaragoza
+Recep_Çetin
+Red_Brigades
+Red_River_of_the_North
+Red_Sea
+Red_Star_Belgrade
+Rede_Globo
+Redmond,_Washington
+Reform_Party_of_the_United_States_of_America
+Regi_Blinker
+Regina_Jacobs
+Reinhard_Schwarzenberger
+Remington_Model_8
+Renate_Götschl
+Rennae_Stubbs
+Rennes
+Renzo_Furlan
+René_(novella)
+René_Eijkelkamp
+René_Schneider_(footballer)
+René_Tretschok
+Repsol_YPF
+Republic_of_China
+Republic_of_Ireland
+Republic_of_Ireland_national_football_team
+Republic_of_Macedonia
+Republic_of_Serbian_Krajina
+Republic_of_the_Congo
+Republican_National_Convention
+Republican_Party_(United_States)
+Republican_Rally_for_Democracy_in_Rwanda
+Republika_(Indonesian_newspaper)
+Republika_Srpska
+Resalat_(newspaper)
+Reserve_Bank_of_Australia
+Reserve_Bank_of_India
+Resistance_movement
+Retief_Goosen
+Reto_Götschi
+Reuters
+Revolutionary_Armed_Forces_of_Colombia
+Revolutionary_United_Front
+Reynald_Pedros
+Rheinpark_Stadion
+Ricardo_Peláez
+Ricardo_Rosset
+Ricardo_Sá_Pinto
+Ricco_Groß
+Richard_Hadlee
+Richard_Hannon
+Richard_Kettleborough
+Richard_Krajicek
+Richard_Lamm
+Richard_Roelofsen
+Richard_Witschge
+Richey_Reneberg
+Richmond,_Virginia
+Richmond_Football_Club
+Richter_magnitude_scale
+Rick_Huisman
+Rickey_Henderson
+Ricky_Ponting
+Ricky_Watters
+Riga
+Rika_Hiraki
+Rio_Tinto_Alcan
+Rio_de_Janeiro
+Riverside_Ground
+Riyadh
+Rob_Andrew
+Rob_Enderle
+Rob_Howley
+Robbie_Earle
+Robbie_Fowler
+Robbie_McEwen
+Robbie_Winters
+Robert_Allenby
+Robert_Coles_(golfer)
+Robert_Croft
+Robert_Karlsson
+Robert_Mugabe
+Robert_P._Casey
+Robert_Pelletreau
+Robert_Pirès
+Robert_Reichel
+Robert_Rubin
+Robert_Samuels
+Robert_Schuman
+Robert_W._Baird
+Roberta_Brunet
+Roberto_Alomar
+Roberto_Ayala
+Roberto_Baggio
+Roberto_Benigni
+Roberto_Bisconti
+Roberto_Carlos_(Spanish_footballer)
+Roberto_Carlos_(footballer)
+Roberto_Carretero
+Roberto_Chiappa
+Roberto_Durán
+Roberto_M._Levingston
+Roberto_Mancini
+Robin_Brooke
+Robin_Ventura
+Rochdale_A.F.C.
+Rochester,_New_Hampshire
+Rocky
+Rocky_Coppinger
+Rod_Beck
+Roda_JC
+Rodney_Eyles
+Roermond
+Rogel_Nachum
+Roger_Black
+Roger_Chapman_(golfer)
+Roger_Clemens
+Roger_García_Junyent
+Roger_Kingdom
+Roger_Pavlik
+Rogers_Cup_(tennis)
+Rohan_Robinson
+Roland_Holder
+Rolf_Ekéus
+Rolf_Fringer
+Rolf_Sørensen
+Roman_Empire
+Romania
+Romania_national_football_team
+Romania_national_under-21_football_team
+Romanian_National_Unity_Party
+Romanians
+Romano_Prodi
+Rome
+Romesh_Kaluwitharana
+Ron_Wright_(baseball)
+Ronald_Brunmayr
+Ronald_Goldman
+Ronald_Hamming
+Ronald_Reagan
+Ronald_Waterreus
+Ronald_de_Boer
+Ronaldo
+Ronan_Rafferty
+Ronde_van_Nederland
+Ronnie_Irani
+Roosevelt_family
+Rory_Underwood
+Rosario,_Santa_Fe
+Rosario_Central
+Rose_Bowl_(stadium)
+Rose_Cheruiyot
+Rosenborg_BK
+Roshan_Mahanama
+Ross_County_F.C.
+Ross_McFarlane
+Ross_Perot
+Rostelecom
+Rotary_International
+Rotherham_United_F.C.
+Rotterdam
+Rottweiler
+Rovereto
+Rowanduz
+Roxbury,_Vermont
+Roy_Aitken
+Roy_Keane
+Royal_Antwerp_FC
+Royal_Boskalis_Westminster
+Royal_Clipper
+Royal_Free_Hospital
+Royal_Meteorological_Institute
+Royal_Tunbridge_Wells
+Royce_Clayton
+Rubber_Board
+Ruben_Kruger
+Rubens_Barrichello
+Rubén_Sierra
+Ruch_Chorzów
+Rudi_Vata
+Rudyard,_Montana
+Rugby_Football_Union
+Rugby_World_Cup
+Rugby_football
+Rugby_league
+Rugby_union
+Ruggiero_Rizzitelli
+Rui_Barros
+Rui_Correia
+Rui_Costa
+Rui_Madeira
+Rumbek
+Run_(baseball)
+Rupert_Murdoch
+Ruse,_Bulgaria
+Russ_Cochran
+Russell_Claydon
+Russia
+Russia_national_football_team
+Russian_Empire
+Russian_Trading_System
+Rusty_Greer
+Ruth_Perry
+Ruxandra_Dragomir
+Rwanda
+Ryder_Cup
+Ryne_Sandberg
+Ryszard_Wieczorek
+Ryutaro_Hashimoto
+Régine_Cavagnoud
+S&P_500
+S.C._Braga
+S.C._Espinho
+S.L._Benfica
+S.S.C._Napoli
+S.S._Lazio
+SBC_Communications
+SBM_Offshore
+SC_Bastia
+SC_Freiburg
+SC_Heerenveen
+SC_Tavriya_Simferopol
+SD_Compostela
+SEGRO
+SFC_Opava
+SIDOR
+SK_Brann
+SK_Dynamo_České_Budějovice
+SK_Rapid_Wien
+SK_Sigma_Olomouc
+SK_Sturm_Graz
+SNCF
+SPAL_1907
+SV_Ried
+SV_Werder_Bremen
+Sabina_Panzanini
+Sabine_Appelmans
+Sabine_Hack
+Sabri_Lamouchi
+Sachin_Tendulkar
+Sacramento_Kings
+Saddam_Hussein
+Saeb_Erekat
+Saeed_Anwar
+Saffet_Sancaklı
+Saint_Croix,_U.S._Virgin_Islands
+Saint_Nicholas
+Sakai,_Osaka
+Salah_Hissou
+Salang,_Afghanistan
+Sale_Grammar_School
+Sale_Sharks
+Saleem_Malik
+Sali_Berisha
+Sally_Barsosio
+Sally_Boyden_(cyclist)
+Salomon_Brothers
+Salzburg
+Sammy_Sosa
+Samson_Kitur
+Samsung_Heavy_Industries
+Samuel_Matete
+San_Antonio_Spurs
+San_Diego
+San_Diego_Chargers
+San_Diego_Padres
+San_Francisco
+San_Francisco_49ers
+San_Francisco_Giants
+San_Jose,_California
+San_Jose_Sharks
+San_José,_Costa_Rica
+San_Lorenzo_de_Almagro
+San_Marino
+San_Marino_national_football_team
+San_Mateo,_California
+Sanath_Jayasuriya
+Sandon_Stolle
+Sandra_Cacic
+Sandra_Cecchini
+Sandra_Kleinová
+Sandrine_Testud
+Sandusky,_Ohio
+Santa_Barbara,_California
+Santa_Claus
+Santiago,_Chile
+Santiago_Bernabéu_Stadium
+Santiago_de_Cuba
+Santos_FC
+Saqlain_Mushtaq
+Saracens_F.C.
+Sarah_Brady
+Sarah_Pitkowski-Malcor
+Sarah_Thorsett
+Sarajevo
+Sardinia
+Saskatchewan
+Saskatchewan_Wheat_Pool
+Satoshi_Higashi
+Sauber
+Saudi_Arabia
+Saudi_Press_Agency
+Saxony
+Scarborough_F.C.
+Schering_AG
+Scotland
+Scotland_national_football_team
+Scotland_national_rugby_union_team
+Scotland_national_under-21_football_team
+Scott_Brosius
+Scott_Draper
+Scott_Erickson
+Scott_Hoch
+Scott_Humphries
+Scott_McCarron
+Scott_McGrory
+Scott_Quinnell
+Scott_Russell_(motorcyclist)
+Scott_Sanders_(baseball)
+Scott_Young_(footballer)
+Scottish_Cup
+Scottish_Highlands
+Scottish_Labour_Party
+Scottish_League_Cup
+Scottish_National_Party
+Scottish_Premier_League
+Scuderia_Ferrari
+Scunthorpe_United_F.C.
+Sean_Berry
+Sean_Dundee
+Sean_Fitzpatrick
+Sean_Olsson
+Seattle
+Seattle_Mariners
+Seattle_Seahawks
+Seattle_SuperSonics
+Sebastian_Lindholm
+Sebastien_Tortelli
+Second_French_Empire
+Security_Council_of_Russia
+Senegal
+Seoul
+Sepp_Dostthaler
+Serbia
+Serbs
+Sergey_Alexandrovich_Makarov
+Sergey_Klevchenya
+Sergey_Yastrzhembsky
+Sergi_Barjuán
+Sergi_Bruguera
+Sergiyev_Posad
+Serhiy_Rebrov
+Serie_A
+Servais_Knaven
+Servet_Pëllumbi
+Servette_FC
+Severiano_Ballesteros
+Sevilla_FC
+Seville
+Seychelles
+Seychelles_national_football_team
+Shadab_Kabir
+Shahid_Afridi
+Shane_Kelly
+Shane_Reynolds
+Shane_Warne
+Shanghai
+Shannon_Airport
+Sharjah_(city)
+Sharjah_(emirate)
+Shatoy
+Shaun_Pollock
+Shaun_Young
+Shawn_Estes
+Shay_Given
+Shayne_King
+Sheffield
+Sheffield_Eagles
+Sheffield_Shield
+Sheffield_United_F.C.
+Sheffield_Wednesday_F.C.
+Sheikh_Hasina
+Shelbourne_F.C.
+Shell_Oil_Company
+Shell_Turbo_Chargers
+Shem_Kororia
+Sherwin_Campbell
+Shigeki_Maruyama
+Shimon_Peres
+Shinichi_Itoh
+Shining_Path
+Shiraz
+Shivnarine_Chanderpaul
+Shrewsbury_Town_F.C.
+Shu_Kamo
+Shuzo_Matsuoka
+Shwedagon_Pagoda
+Sialkot
+Siam_Commercial_Bank
+Sibiu_International_Airport
+Sicily
+Sidwell_Friends_School
+Sidya_Touré
+Sierakowice,_Pomeranian_Voivodeship
+Sierra_Leone
+Sierra_Leone_national_football_team
+Sierra_Nevada_(U.S.)
+Sigurd_Njerve
+Silicon_Valley
+Silke_Renk
+Silvia_Farina_Elia
+Silvio_Berlusconi
+Simon_Brown_(cricketer)
+Simon_Crafar
+Simon_Culhane
+Simon_Doull
+Simon_Parke
+Simone_Greiner-Petter-Memm
+Sinaloa
+Sindh_High_Court
+Singapore
+Singapore_International_Monetary_Exchange
+Singer_World_Series
+Sisters_of_Loreto
+Sivas
+Six_Nations_Championship
+Sjeng_Schalken
+Skeid_Fotball
+Skeleton_Canyon
+Skhirat
+Skopje
+Sky_Digital_(UK_&_Ireland)
+Slavia_Prague
+Sliema
+Slobodan_Milošević
+Slovakia
+Slovakia_national_football_team
+Slovenia
+Small_Is_Beautiful
+Social_Democratic_Party_(Japan)
+Social_Democratic_Party_(Romania)
+Social_Democratic_Party_of_Germany
+Socialist_Federal_Republic_of_Yugoslavia
+Socialist_Party_of_Albania
+Socialist_Party_of_Serbia
+Sociedade_Esportiva_Palmeiras
+Sofia
+SoftBank
+Sokoto
+Solidere
+Somalia
+Somerset_County_Cricket_Club
+Son_Sen
+Sonora
+Sophia_Loren
+Sourav_Ganguly
+South_Africa
+South_Africa_national_cricket_team
+South_Africa_national_field_hockey_team
+South_Africa_national_rugby_union_team
+South_African_Breweries
+South_America
+South_China_Morning_Post
+South_Dakota
+South_Island
+South_Kivu
+South_Korea
+South_Lebanon_Army
+South_Queensland_Crushers
+South_Sydney_Rabbitohs
+South_Yemen
+Southampton
+Southampton_F.C.
+Southeast_Asia
+Southend_United_F.C.
+Southern_California_Open
+Southern_Illinois_University_Carbondale
+Southern_New_England_Telecommunications
+Soviet_Union
+Spain
+Spain_Fed_Cup_team
+Spanish_language
+Spanish_people
+Sparta_Prague
+Sparta_Rotterdam
+Speed_Skating_World_Cup
+Spetses
+Spirou_Charleroi
+Split_(city)
+Sport_Club_Corinthians_Paulista
+Sport_Club_Internacional
+Sport_Club_do_Recife
+Sporting_Clube_de_Portugal
+Sporting_de_Gijón
+Sprague,_Manitoba
+Springfield,_Massachusetts
+Srebrenica
+Sri_Lanka
+Sri_Lanka_national_cricket_team
+Sri_Lankan_Tamil_people
+Srinagar
+St._Albans_(city),_Vermont
+St._Gallen
+St._George_Dragons
+St._George_Illawarra_Dragons
+St._Johnstone_F.C.
+St._Leger_Stakes
+St._Louis,_Missouri
+St._Louis_Blues_(ice_hockey)
+St._Louis_Cardinals
+St._Mirren_F.C.
+St._Petersburg,_Florida
+St_Albans_City_F.C.
+St_Helens,_Merseyside
+St_Helens_RLFC
+St_Kilda_Football_Club
+Stabæk_Fotball
+Stade_Malherbe_Caen
+Stade_Rennais_F.C.
+Stagecoach_Group
+Standard_&_Poor's
+Standard_Liège
+Stansted_Mountfitchet
+Staouéli
+State_Council_of_the_People's_Republic_of_China
+State_of_Palestine
+Statistics_Canada
+Statoil
+Stefaan_De_Clerck
+Stefan_Edberg
+Stefan_Johansson
+Stefan_Kuntz
+Stefan_Reuter
+Stefan_Schwarz
+Stefano_Pescosolido
+Stefano_Tilli
+Stefano_Zanini
+Steffen_Freund
+Steffi_Graf
+Stefka_Kostadinova
+Stenhousemuir_F.C.
+Stephan_Marasek
+Stephanie_Storp
+Stephen_Ames
+Stephen_Brown_(athlete)
+Stephen_McAllister
+Stephen_Pate
+Stephenville_International_Airport
+Steptoe_&_Johnson
+Sterling_Hitchcock
+Steve_Backley
+Steve_Finley
+Steve_Howey_(footballer)
+Steve_Jones_(golfer)
+Steve_McManaman
+Steve_Redgrave
+Steve_Staunton
+Steve_Stricker
+Steve_Waugh
+Steve_Webster
+Steve_van_Vuuren
+Stevenage_F.C.
+Stewart_Ginn
+Steyr
+Stirling
+Stirling_Albion_F.C.
+Stockholm
+Stockport_County_F.C.
+Stoke_City_F.C.
+Straits_of_Florida
+Stranraer
+Stranraer_F.C.
+Strasbourg
+Strobe_Talbott
+Stuart_Appleby
+Stuart_Cage
+Stuart_Law
+Stuart_McCall
+Stung_Treng
+Stuttgart
+Stuttgart_Airport
+Stéphane_Chapuisat
+Stéphane_Guivarc'h
+Stéphane_Henchoz
+Stéphane_Ortelli
+Stéphane_Simian
+Subaru
+Subaru_Impreza
+Subaru_Impreza_WRX_STI
+Subaru_World_Rally_Team
+Sud-PTT
+Sudan
+Sudan_Airways
+Sudan_national_football_team
+Sudbury_Town_F.C.
+Suharto
+Sui_Southern_Gas_Company
+Sulaymaniyah
+Sultan,_Crown_Prince_of_Saudi_Arabia
+Summer_Olympic_Games
+SunGard
+Sun_Jun_(badminton)
+Sun_Microsystems
+Sunday_Bada
+Sunderland_A.F.C.
+Super_League
+Supercopa_de_España
+Supercoppa_Italiana
+Superman_(film)
+Supreme_Court_of_Florida
+Supreme_Court_of_New_South_Wales
+Supreme_Court_of_the_Republic_of_China
+Supreme_Court_of_the_United_States
+Supreme_Federal_Court
+Surabaya
+Suraj_Bhan
+Suriname
+Surrey
+Surrey_County_Cricket_Club
+Surčin
+Susan_Roosevelt_Weld
+Susi_Susanti
+Sussex_County_Cricket_Club
+Suwon_Samsung_Bluewings
+Suzuka_Circuit
+Suzuki
+Sven_Fischer
+Sven_Nylander
+Sven_Pieters
+Sven_Strüver
+Svetlana_Masterkova
+Svetlana_Paramygina
+Svetlana_Zalevskaya
+Swansea_City_A.F.C.
+Swansea_RFC
+Sweden
+Sweden_men's_national_ice_hockey_team
+Sweden_national_football_team
+Swerford
+Swindon_Town_F.C.
+Swiss_Bank_Corporation
+Switzerland
+Switzerland_national_football_team
+Sydney
+Sydney_Cricket_Ground
+Sydney_Roosters
+Sydney_Swans
+Sylvain_Bouchard
+Sylvester_Stallone
+Syria
+Syria_national_football_team
+São_Paulo
+São_Paulo_(state)
+São_Paulo_FC
+Sébastien_Foucras
+Sébastien_Fournier
+Sébastien_Lareau
+Sérgio_Conceição
+Sören_Lausberg
+Søren_Jessen-Petersen
+Süddeutsche_Zeitung
+Süleyman_Demirel
+Sławomir_Wojciechowski
+T-72
+TSV_1860_München
+TVM_(cycling_team)
+TV_Guide
+Ta_Mok
+Tadayuki_Okada
+Tae_Satoya
+Taibe,_Galilee
+Taipei
+Taiwan
+Taiwan_Strait
+Tajikistan
+Takanobu_Okabe
+Takuma_Aoki
+Takuya_Takagi
+Taliban
+Tallahassee,_Florida
+Tallinn
+Tamaulipas
+Tami_Whitlinger
+Tamil_Eelam_Liberation_Organization
+Tamil_Nadu
+Tamil_people
+Tampa_Bay_Buccaneers
+Tampa_Bay_Lightning
+Tampere
+Tampico,_Tamaulipas
+Tando_Allahyar
+Tang_Jiaxuan
+Tanja_Damaske
+Tanjug
+Tanjung_Priok
+Tansu_Çiller
+Tanya_Dubnicoff
+Tanzania
+Tanzania_national_football_team
+Tarek_Jabban
+Tariq_Aziz
+Tarmac_(company)
+Tarpon_Springs,_Florida
+Tartus
+Tasmania
+Tasmanian_Tigers
+Tatarstan
+Tatiana_Stiajkina
+Tatjana_Mittermayer
+Tatyana_Babashkina
+Tauranga
+Tavildara
+Tayfun_Korkut
+Tea_Vikstedt-Nyman
+Team_Polti
+Teddy_Sheringham
+Teemu_Selänne
+Tegla_Loroupe
+Tegucigalpa
+Tehran
+Tel_Aviv
+Telegraph_Agency_of_the_Soviet_Union
+Televisa
+Television_New_Zealand
+Telfer_Mine
+Telkom_Indonesia
+Telmex
+Telstra
+Tempe,_Arizona
+Ten_Commandments
+Tennessee
+Tennessee_Titans
+Teresa
+Terry_Kennedy
+Terry_Mulholland
+Terry_Pendleton
+Terry_Phelan
+Terry_Price_(golfer)
+Terry_Steinbach
+Texas
+Texas_Panhandle
+Texas_Rangers_(baseball)
+Thabo_Mbeki
+Thai_Nation_Party
+Thailand
+The_Ashes
+The_Bahamas
+The_Bank_of_Tokyo-Mitsubishi_UFJ
+The_Conference_Board
+The_Crying_Game
+The_Daily_Telegraph_(Australia)
+The_Hague
+The_Holocaust
+The_Irish_Times
+The_Jakarta_Post
+The_Jerusalem_Post
+The_Jordan_Times
+The_Lancet
+The_Land_is_Ours
+The_Miami_Herald
+The_Nation
+The_New_Saints_F.C.
+The_New_York_Times
+The_Open_Championship
+The_Oval
+The_Pentagon
+The_Statesman
+The_Sumitomo_Bank
+The_Sun_(United_Kingdom)
+The_Sunday_Telegraph
+The_Times
+The_Vancouver_Sun
+The_Wall_Street_Journal
+The_Walt_Disney_Company
+The_Washington_Post
+Theodore_Roosevelt
+Theodoros_Pangalos
+Thessaloniki
+Thierry_Henry
+Thomas_Bjørn
+Thomas_Bscher
+Thomas_Enqvist
+Thomas_Helmer
+Thomas_Häßler
+Thomas_Johansson
+Thomas_Muster
+Thomas_Rådström
+Thomas_Seeliger
+Thomas_Sobotzik
+Thomas_Stickroth
+Thomas_Strunz
+Thomas_Stuer-Lauridsen
+Thunder_Bay
+Tiananmen_Square
+Tianjin
+Tibet
+Tiburones_Rojos_de_Veracruz
+Tiger_Woods
+Tignes
+Tijani_Babangida
+Tijuana
+Tim_Belcher
+Tim_Forsyth
+Tim_Hancock
+Tim_Henman
+Tim_Herron
+Tim_Horan
+Tim_Lobinger
+Tim_Munton
+Tim_Simpson
+Tim_Stimpson
+Tim_Wakefield
+Timişoara
+Tina_Križan
+Tino_Martinez
+Tirana
+Todd_Hollandsworth
+Todd_Martin
+Todd_Stottlemyre
+Todd_Van_Poppel
+Todd_Woodbridge
+Todd_Worrell
+Todor_Zhivkov
+Toftir
+Togo
+Togo_national_football_team
+Tokyo
+Tokyo_Stock_Exchange
+Toledo,_Ohio
+Tolunay_Kafkas
+Tom_Boyd_(footballer)
+Tom_Candiotti
+Tom_Cruise
+Tom_Daschle
+Tom_Glavine
+Tom_Ikimi
+Tom_Johnson_(boxer)
+Tom_Kiernan
+Tom_Lehman
+Tom_Moody
+Tom_Nyariki
+Tom_Pagnozzi
+Tom_Pukstys
+Tom_Steels
+Tom_Watson_(golfer)
+Tomasz_Moskal
+Tommy_Haas
+Tommy_Martyn
+Tommy_Thompson
+Tomás_Carbonell
+Tonga
+Toni_Polster
+Tony_Adams_(footballer)
+Tony_Blair
+Tony_Cascarino
+Tony_Clark
+Tony_Eusebio
+Tony_Greig
+Tony_Jarrett
+Tony_Johnstone
+Tony_P._Hall
+Tony_Roche
+Tony_Siragusa
+Tony_Underwood
+Tony_Vairelles
+Toomas_Savi
+Topoľčany
+Tops_Markets_LLC
+Torino_F.C.
+Toronto
+Toronto_Blue_Jays
+Toronto_Maple_Leafs
+Toronto_Raptors
+Toros_Neza
+Torquay_United_F.C.
+Torrance_Zellner
+Tottenham_Hotspur_F.C.
+Tour_de_France
+Tour_de_Suisse
+Towers_Perrin
+Toyota
+Toyota_Australia
+Toyota_Celica
+Tranmere_Rovers_F.C.
+Trans_World_Airlines
+Transcendental_Meditation
+Transvaal_Province
+Transylvania
+Travis_Fryman
+Treaty_of_Baden
+Trenidad_Hubbard
+Trent_Bridge
+Treorchy_RFC
+Trevor_Dodds
+Trincomalee
+Trinidad
+Tripoli,_Lebanon
+Tristan_Hoffman
+Troy,_Tennessee
+Troy_Corser
+Troy_Vincent
+Tubmanburg
+Tunceli_Province
+Tunis
+Tunisia
+Tunisia_national_football_team
+Turin
+Turkey_national_football_team
+Turkish_Armed_Forces
+Turkish_Cypriots
+Turnhout
+Tuscany
+Tutsi
+Tuzla
+Twickenham_Stadium
+Twyford_Down
+Ty_Detmer
+Tychy
+Tyler_Houston
+Tyrrell_Racing
+Tórshavn
+Túlio_Costa
+U.C._Sampdoria
+U.S._Bank_Championship_in_Milwaukee
+U.S._Città_di_Palermo
+U.S._Cremonese
+U.S._Lecce
+U.S._Open_(golf)
+U._Chandana
+UBS
+UCI_Road_World_Cup
+UD_Salamanca
+UEFA
+UEFA_Champions_League
+UEFA_Cup_Winners'_Cup
+UEFA_Euro_1992
+UEFA_Euro_1996
+UEFA_Europa_League
+UEFA_European_Football_Championship
+UEFA_Intertoto_Cup
+UNITA
+USS_Carl_Vinson_(CVN-70)
+USS_Enterprise_(CVN-65)
+USTA_Billie_Jean_King_National_Tennis_Center
+US_Airways
+US_Open_(tennis)
+Udinese_Calcio
+Uganda
+Uganda_national_football_team
+Ujjain
+Ukraine
+Ukraine_national_football_team
+Ukyo_Katayama
+Ulan_Bator
+Ulf_Kirsten
+Ulsan
+Ulsan_Hyundai_FC
+Ulysses_S._Grant
+Umberto_Bossi
+Umbria
+Umkomaas
+Umm_Qasr
+Unified_Communist_Party_of_Nepal_(Maoist)
+Union_Bank_of_Switzerland
+Union_Luxembourg
+United_Arab_Emirates
+United_Arab_Emirates_national_football_team
+United_Daughters_of_the_Confederacy
+United_Kingdom
+United_Liberation_Movement_of_Liberia_for_Democracy_–_Johnson_faction
+United_National_Party
+United_Nations
+United_Nations_Commission_on_Human_Rights
+United_Nations_High_Commissioner_for_Refugees
+United_Nations_Interim_Force_in_Lebanon
+United_Nations_Security_Council
+United_Nations_Special_Commission
+United_News_of_India
+United_States
+United_States_Air_Force
+United_States_Amateur_Championship_(golf)
+United_States_Army
+United_States_Congress
+United_States_Court_of_Appeals_for_the_Fourth_Circuit
+United_States_Department_of_Agriculture
+United_States_Department_of_Commerce
+United_States_Department_of_Defense
+United_States_Department_of_State
+United_States_Department_of_the_Treasury
+United_States_Fed_Cup_team
+United_States_Marine_Corps
+United_States_Navy
+United_States_Senate_Committee_on_Armed_Services
+United_States_Senate_Select_Committee_on_Intelligence
+United_States_Tennis_Association
+United_States_Treasury_security
+United_States_courts_of_appeals
+United_States_district_court
+United_States_dollar
+United_States_men's_national_soccer_team
+University_of_Michigan
+University_of_Oklahoma
+University_of_Oxford
+University_of_Pennsylvania
+University_of_Yangon
+University_of_the_Witwatersrand
+Unión_de_Santa_Fe
+Uppsala
+Uppsala_University_Hospital
+Upul_Chandana
+Ural_Mountains
+Uruguay
+Urus-Martan
+Utah
+Utah_Jazz
+Uttar_Pradesh
+Uyo
+Uzbekistan
+Uzbekistan_national_football_team
+Uzi
+V.C._Eendracht_Aalst_2002
+Vadim_Sashurin
+Vaduz
+Vail,_Colorado
+Valdo_Filho
+Vale_(mining_company)
+Valencia,_Spain
+Valencia_CF
+Valentina_Fedyushina
+Valero_Energy_Corporation
+Valero_Texas_Open
+Valletta
+Valletta_F.C.
+Vampeta
+Van,_Turkey
+Vancouver
+Vancouver_Canucks
+Vanderbijlpark
+Variety_(magazine)
+Vasas_SC
+Vatican_City
+Vatican_Library
+Vavuniya
+Veliko_Tarnovo
+Velten
+Venezuela
+Venice
+Venice_Film_Festival
+Venkatesh_Prasad
+Venray
+Ventura,_California
+VeriSign
+Verona
+Veronica_Brenner
+VfB_Stuttgart
+VfL_Bochum
+Viacom_(1971–2005)
+Viatcheslav_Ekimov
+Vicente_Solano_Lima
+Vicenza_Calcio
+Victor_Babeş
+Victoria_(Australia)
+Victoria_Falls,_Zimbabwe
+Victorian_Bushrangers
+Vidadi_Rzayev
+Videoton_FC
+Vienna
+Vietnam
+Vietnam_War
+Viking_FK
+Viktor_Chernomyrdin
+Viktor_Paço
+Ville_Peltonen
+Vince_Lombardi
+Vince_Spadea
+Vince_Wells
+Vinko_Marinović
+Vinny_Castilla
+Vinnytsia
+Vinod_Kambli
+Violeta_Chamorro
+Viorel_Ion
+Viorel_Moldovan
+Virakesari
+Virgilijus_Alekna
+Virginia
+Virginia_Ruano_Pascual
+Virginia_Tech_Hokies_football
+Virtus_Bologna
+Vitali_Smirnov
+Vitesse
+Vitória_S.C.
+Viva_Zapata!
+Vladimir_Dubrovshchik
+Vladimir_Zhirinovsky
+Vladimír_Mečiar
+Vladimír_Šmicer
+Vogue_(magazine)
+Volker_Rühe
+Volkswagen
+Volkswagen_Group
+Volkswagen_Passat
+Vtm
+Václav_Havel
+Václav_Klaus
+Vålerenga_Fotball
+Vénuste_Niyongabo
+Víctor_Sánchez
+Vítor_Baía
+WA_Reds
+WBUR
+WGC-Bridgestone_Invitational
+WSC_World_XI
+Waiau,_Canterbury
+Waitrose
+Wales
+Wales_national_football_team
+Wales_national_rugby_union_team
+Wales_national_under-21_football_team
+Walikale
+Wall_Street
+Wally_Joyner
+Wally_Whitehurst
+Walsall_F.C.
+Walter_Little_(rugby_player)
+Walter_Mondale
+Wang_Chen_(badminton)
+Wang_Shi-ting
+Waqar_Younis
+War_of_the_Spanish_Succession
+Warner_Bros.
+Warren_Buffett
+Warrington_Wolves
+Warsaw
+Warsaw_Pact
+Warwickshire_County_Cricket_Club
+Washington,_D.C.
+Washington_(U.S._state)
+Washington_Capitals
+Washington_Redskins
+Washington_Wizards
+Wasim_Akram
+Wataru_Yoshikawa
+Waterville,_Washington
+Watford_F.C.
+Watsonians_RFC
+Wayne_Ferreira
+Wayne_Riley
+Wayne_Westner
+Wednesday
+Wellington
+Wembley_Arena
+Wembley_Stadium
+Wen_Hui_Bao
+Wen_Wei_Po
+Wenatchee,_Washington
+Wenchang
+Wendy_Everson
+West_Azarbaijan_province
+West_Bengal
+West_Bromwich_Albion_F.C.
+West_Coast_Eagles
+West_Ham_United_F.C.
+West_Hartlepool_R.F.C.
+West_Indies_cricket_team
+West_South_Central_States
+West_Virginia
+Western_Australia
+Western_Bulldogs
+Western_Hemisphere
+Western_Province_cricket_team
+Western_Suburbs_Magpies
+Western_world
+Weston-super-Mare
+Whistler,_British_Columbia
+Whistler_Mountain
+White_House
+Whitehill_Welfare_F.C.
+Whitewater_controversy
+Whittier,_Alaska
+Widnes_Vikings
+Widzew_Łódź
+Wigan
+Wigan_Athletic_F.C.
+Wigan_Warriors
+Wilfried_Peeters
+Wilfried_Van_Moer
+Wilhelmina_of_the_Netherlands
+Will_Carling
+Willem_II_(football_club)
+William_Goldman
+William_Hill_(bookmaker)
+William_Jordan,_Baron_Jordan
+William_Perry
+William_Tanui
+William_Weld
+WilliamsF1
+Willie_Wood_(golfer)
+Wilson_Kipketer
+Wiltshire
+Wim_Jonk
+Wimbledon_Championships
+Wimbledon_F.C.
+Wincanton_Classic
+Windhoek
+Windows_95
+Windows_NT
+Windows_NT_4.0
+Winnipeg
+Winston_Bogarde
+Winston_Peters
+Wisconsin
+Wisła_Kraków
+Woking_F.C.
+Wolf_Point,_Montana
+Wolfgang_Feiersinger
+Wolfgang_Ischinger
+Wolverhampton_Wanderers_F.C.
+Woodridge,_Illinois
+Worcester
+Worcestershire_County_Cricket_Club
+Workington_Town
+World_Badminton_Grand_Prix
+World_Boxing_Organization
+World_Cup_of_Hockey
+World_Open
+World_Series
+World_Series_Cricket
+World_Series_of_Golf
+World_Trade_Organization
+World_War_II
+World_Wide_Web
+World_championship
+Worthing
+Wrexham_F.C.
+Wrocław
+Wuhan
+Wuxi
+Wyche_Fowler
+Wycombe_Wanderers_F.C.
+Wydad_Casablanca
+Xavier_Gravelaine
+Xhosa
+Xiao_Qiang
+Xinhua_News_Agency
+Xiomara_Rivero
+Xu_Nannan
+YMCA
+Yakoma_people
+Yamaha_Motor_Company
+Yangon
+Yangon_Technological_University
+Yap_Kim_Hock
+Yasser_Arafat
+Yassine_Abdellaoui
+Yasushi_Akashi
+Yasuto_Honda
+Yayuk_Basuki
+Ye_Zhaoying
+Yedioth_Ahronoth
+Yegor_Stroyev
+Yekaterina_Podkopayeva
+Yelena_Gulyayeva
+Yemen
+Yeosu
+Yerevan
+Yevgeny_Kafelnikov
+Yevgeny_Primakov
+Yingkou
+Yitzhak_Mordechai
+Yitzhak_Shamir
+Yoelbi_Quesada
+Yonhap
+Yonsei_University
+York
+York_City_F.C.
+Yorkshire
+Yorkshire_County_Cricket_Club
+Younes_El_Aynaoui
+Youri_Djorkaeff
+Youri_Mulder
+Yugoslavia
+Yukio_Hatoyama
+Yunnan
+Yuriy_Nikiforov
+Yvan_Quentin
+Yvonne_McGregor
+ZDF
+Zaandam
+Zabrze
+Zadar
+Zagreb
+Zagłębie_Lubin
+Zahoor_Elahi
+Zalaegerszegi_TE
+Zambia
+Zambia_national_football_team
+Zapatista_Army_of_National_Liberation
+Zastava_Arms
+Zbigniew_Siemiątkowski
+Zdeněk_Svoboda
+Zelimkhan_Yandarbiyev
+Zenith_Electronics
+Zevenaar
+Zhang_Ning
+Zhanna_Pintusevich-Block
+Zhejiang
+Zimbabwe
+Zimbabwe_Open
+Zimbabwe_national_cricket_team
+Zimbabwe_national_football_team
+Zina_Garrison
+Zine_El_Abidine_Ben_Ali
+Zinedine_Zidane
+Zinzan_Brooke
+Zionism
+Zoran_Savić
+Zulu_Kingdom
+Zvornik
+Zygmunt_Solorz-Żak
+Zëri_i_Popullit
+Zürich
+Àlex_Corretja
+Àlex_Crivillé
+Álvaro_Espinoza
+Ángeles_Montolio
+Åsa_Svensson
+Écu
+Élan_Béarnais_Pau-Orthez
+Éric_Bernard
+Óscar_Tabárez
+Östersund
+Újpest_FC
+Čelopek
+Ľubomír_Moravčík
+ŁKS_Łódź
+Śląsk_Wrocław
+Şırnak_Province
+ŠK_Slovan_Bratislava
+Šiauliai
+Šárka_Kašpárková
+Życie_Warszawy
+Željko_Petrović

span_annotation.py

@@ -0,0 +1,166 @@
+"""
+Data structure classes required and used for multiple levels of granularity in spans.
+"""
+from data_loader import dl_sa
+from mosestokenizer import MosesDetokenizer
+detokenize = MosesDetokenizer('en')
+
+
+class PhraseAnnotation:
+    def __init__(self, initial_word):
+        self.words = [initial_word]
+        self._resolved_annotation = initial_word.resolved_annotation
+        self.ppr_for_ned_candidates = initial_word.ppr_for_ned_candidates
+
+    @property
+    def has_valid_bioes_labels(self):
+        # B = 0, I = 1, O = 2, E = 3, S = 4
+        return all([x.has_valid_bioes_labels and x.bioes_labels is not None for x in self.words])
+
+    def add(self, word):
+        self.words.append(word)
+        # There are some phrases that are annotated as O but have PPRforNED candidates, those will be ignored!
+        if self._resolved_annotation > 0 and self.ppr_for_ned_candidates != word.ppr_for_ned_candidates:
+            self.ppr_for_ned_candidates = list(set(self.ppr_for_ned_candidates) & set(word.ppr_for_ned_candidates))
+
+    def all_possible_annotations(self):
+        all_common_ids = set.intersection(*[set([y[0] for y in x.candidates]) for x in self.words])
+        all_common_ids_average_confidence = map(lambda x: sum(x)/len(x), [
+            [sum(y[1])/len(y[1]) for x in self.words for y in x.candidates if y[0] == k] for k in all_common_ids])
+        return sorted(zip(all_common_ids, all_common_ids_average_confidence), key=lambda x: x[1], reverse=True)
+
+    def set_alternative_as_resolved_annotation(self, alternative):
+        self._resolved_annotation = alternative
+
+    @property
+    def resolved_annotation(self):
+        return self._resolved_annotation
+
+    @property
+    def subword_annotations(self):
+        return [x for w in self.words for x in w.annotations]
+
+    @property
+    def word_string(self):
+        return detokenize([x.word_string.replace("\n", "\u010a").replace("£", "£").replace("âĦ¢", '™')
+                          .replace('ü','ü').replace('é', 'é').replace('ÃŃ', 'í') for x in self.words])
+
+    @property
+    def begin_character(self):
+        return self.words[0].token_offsets[0][1][0]
+
+    @property
+    def end_character(self):
+        return self.words[-1].token_offsets[-1][1][-1]
+
+    @property
+    def average_annotation_confidence(self):
+        ac = [x.resolved_annotation_confidence for x in self.words]
+        return sum(ac) / len(ac)
+
+    def __str__(self):
+        return f"{self.word_string} ({self.begin_character}, {self.end_character}) | annotation: " \
+               f"{self.words[0].annotations[0].idx2tag[self.resolved_annotation]}"
+
+
+class WordAnnotation:
+    def __init__(self, subword_annotations, token_offsets, ppr_for_ned_candidates=None):
+        if ppr_for_ned_candidates is None:
+            ppr_for_ned_candidates = []
+        self.annotations = subword_annotations
+        self.token_offsets = token_offsets
+        self.ppr_for_ned_candidates = ppr_for_ned_candidates
+        self.is_valid_annotation = False if not subword_annotations else True
+        self.word_string = ''.join([x[0].replace('\u0120', '') for x in token_offsets])
+        # even if self.is_valid_annotation is True we could still have the candidates to be empty
+        #   since there could be no consensus among the subword predictions.
+        self.candidates = sorted([] if not self.is_valid_annotation else [
+            (cid, self._get_assigned_probabilities(cid)) for cid in set.intersection(*[set(y.top_k_i_list)
+                                                                                       for y in self.annotations])],
+                                 key=lambda x: sum(x[1])/len(x[1]), reverse=True)
+        self.resolved_annotation = self._resolve_annotation()
+        rc = self._get_assigned_probabilities(self.resolved_annotation)
+        self.resolved_annotation_confidence = sum(rc) / len(rc)
+        if not self.candidates:
+            self.candidates = [(self.resolved_annotation, rc)]
+        assert self.resolved_annotation in [x[0] for x in self.candidates]
+        self.has_valid_bioes_labels = all([x.has_valid_bioes_label for x in self.annotations])
+        self.bioes_labels = None if not self.has_valid_bioes_labels else [x.bioes_label for x in self.annotations]
+
+    def _resolve_annotation(self):
+        if not self.is_valid_annotation:
+            return 0
+        r = [x.item() for x in self.annotations]
+        if r.count(r[0]) == len(r):
+            annotation = r[0]
+        elif self.candidates:
+            # here we return the annotation with the highest average probability prediction over all the subwords
+            annotation = self.candidates[0][0]
+        else:
+            # here we return the annotation which the model has predicted as highest probability for
+            #   the majority of the subwords
+            most_frequent = max(set(r), key=r.count)
+            if r.count(most_frequent) == 1:
+                annotation = r[0]
+            else:
+                annotation = most_frequent
+        return annotation
+
+    def _get_assigned_probabilities(self, cid):
+        assigned_probabilities = []
+        for a in self.annotations:
+            found = False
+            for i, p in zip(a.top_k_i_list, a.top_k_p_list):
+                if i == cid:
+                    assigned_probabilities.append(p)
+                    found = True
+                    break
+            if not found:
+                assigned_probabilities.append(0.0)
+        assert len(assigned_probabilities) == len(self.annotations)
+        return assigned_probabilities
+
+    def __str__(self):
+        ann = self.annotations[0].idx2tag[self.resolved_annotation]
+        cdns = ','.join([f'({self.annotations[0].idx2tag[x[0]]}: {sum(x[1])/len(x[1])})' for x in self.candidates])
+        return f"{self.word_string} | annotation: {ann} | candidates: [{cdns}]"
+
+
+class SubwordAnnotation:
+    """
+    The value of his class will be equal to the value of its "self.top_k_i_list[0]", the rest of the information will be
+     carried over for future decision-making and evaluation.
+    """
+    def __init__(self, top_k_p_list, top_k_i_list, subword_string):
+        self.top_k_p_list = top_k_p_list
+        self.top_k_i_list = top_k_i_list
+        subword_string = "UNDEF_STR" if not subword_string else subword_string
+        self.subword_string = subword_string.replace('\u0120', '')
+        self.bioes_label = 2
+        self.has_valid_bioes_label = False
+        self.bioes_probabilities = None
+
+    def __eq__(self, other):
+        if isinstance(other, int):
+            return self.top_k_i_list[0] == other
+        elif isinstance(other, SubwordAnnotation):
+            return self.top_k_i_list[0] == other.top_k_i_list[0]
+        else:
+            raise ValueError
+
+    def __str__(self):
+        return f"({self.subword_string}, <<" \
+               f"{'>> <<'.join([f'{dl_sa.mentions_itos[i]}: {p:.3f}' for i, p in zip(self.top_k_i_list, self.top_k_p_list)])}>>)"
+
+    def item(self):
+        return self.top_k_i_list[0]
+
+    def item_probability(self):
+        return self.top_k_p_list[0]
+
+    def set_bioes_label(self, label: int, probs: list):
+        assert 0 <= label <= 5
+        assert len(probs) == 5
+        self.has_valid_bioes_label = True
+        self.bioes_label = label
+        self.bioes_probabilities = probs

utils.py

@@ -0,0 +1,545 @@
+"""
+This file provides many functionalities that can be shared among different components.
+The most important function in this file is `chunk_annotate_and_merge_to_phrase` which recieves a model and a raw text,
+ annotates the text, and returns the annotation spans.
+"""
+import os
+import json
+import pickle
+import string
+from enum import Enum
+from tqdm import tqdm
+
+from data_loader import get_dataset, tokenizer, dl_sa
+from span_annotation import SubwordAnnotation, WordAnnotation, PhraseAnnotation
+from aida import AIDADataset
+from configuration import get_resources_dir
+from mosestokenizer import MosesTokenizer, MosesPunctuationNormalizer
+moses_tokenize = MosesTokenizer('en', old_version=True)
+normalize = MosesPunctuationNormalizer('en')
+
+
+def get_punc_tokenized_words_list(word_list: list, labels_list: list = None):
+    tokens = []
+    labels = []
+    for w_ind, o_token in enumerate(word_list):
+        if o_token[0] not in string.punctuation and o_token[-1] not in string.punctuation:
+            tokens.append(o_token)
+            if labels_list:
+                labels.append(labels_list[w_ind])
+            if o_token.endswith("\'s") or o_token.endswith("\'S"):
+                tokens[-1] = tokens[-1][:-2]
+                tokens.append(o_token[-2:])
+                if labels_list:
+                    labels.append(labels_list[w_ind])
+            continue
+        # cases where the tokens start or end with punctuation
+        before_tokens = []
+        after_tokens = []
+        while o_token and o_token[0] in string.punctuation:
+            before_tokens.append(o_token[0])
+            o_token = o_token[1:]
+        while o_token and o_token[-1] in string.punctuation:
+            after_tokens.append(o_token[-1])
+            o_token = o_token[:-1]
+        if before_tokens:
+            tokens.append("".join(before_tokens))
+            if labels_list:
+                labels.append(labels_list[w_ind])
+        if o_token:
+            tokens.append(o_token)
+            if labels_list:
+                labels.append(labels_list[w_ind])
+        if after_tokens:
+            tokens.append("".join(after_tokens[::-1]))
+            if labels_list:
+                labels.append(labels_list[w_ind])
+    if labels_list:
+        return tokens, labels
+    return tokens
+
+
+def save_predictions_result(logdir, epoch, precision, recall, f1, num_proposed, num_correct, num_gold,
+                            all_words, all_tags, all_y_hat, all_predicted):
+    final = logdir + "/%s.P%.2f_R%.2f_F%.2f" % ("{}".format(str(epoch)), precision, recall, f1,)
+    with open(final, "w") as fout:
+        for words, tags, y_hat, preds in zip(all_words, all_tags, all_y_hat, all_predicted):
+            assert len(preds) == len(words) == len(tags)
+            for w, t, p in zip(words, tags, preds):
+                if w == '<s>' or t == '<pad>':
+                    continue
+                fout.write(f"{w}\t{t}\t{p}\n")
+            fout.write("\n")
+        fout.write(f"num_proposed={num_proposed}\n")
+        fout.write(f"num_correct={num_correct}\n")
+        fout.write(f"num_gold={num_gold}\n")
+        fout.write(f"precision={precision}\n")
+        fout.write(f"recall={recall}\n")
+        fout.write(f"f1={f1}\n")
+
+
+def get_subword_to_word_mapping(subword_tokens, original_string, sequence_starts_and_ends_with_bos_eos=True):
+    # subword_tokens starts with <s> and ends with </s>
+    if sequence_starts_and_ends_with_bos_eos:
+        subword_tokens = subword_tokens[1:-1]
+    subword_to_word_mapping = []
+    start_subword_index = 0
+    end_subword_index = 0
+    original_tokens = get_punc_tokenized_words_list(original_string.split())
+
+    original_pointer = 0
+    while len(subword_to_word_mapping) != len(original_tokens):
+        next_t = tokenizer.convert_tokens_to_string(subword_tokens[start_subword_index:end_subword_index])
+        next_t = next_t.strip()
+        if next_t == original_tokens[original_pointer]:
+            subword_to_word_mapping.append((start_subword_index, end_subword_index))
+            original_pointer += 1
+            start_subword_index = end_subword_index
+        else:
+            end_subword_index += 1
+        if end_subword_index - start_subword_index > 1000:
+            for i in [0, 1, 2, 3, 4]:
+                n = tokenizer.convert_tokens_to_string(subword_tokens[start_subword_index:start_subword_index + 2 + i]).strip()
+                o = "".join(original_tokens[original_pointer: original_pointer + 2]).replace("`", "\'")
+                if n == o or n.replace(" ", "") == o.replace(" ", ""):
+                    subword_to_word_mapping.append((start_subword_index, start_subword_index + 1))
+                    original_pointer += 1
+                    start_subword_index = start_subword_index + 1
+                    subword_to_word_mapping.append((start_subword_index, start_subword_index + 1 + i))
+                    original_pointer += 1
+                    start_subword_index = start_subword_index + 1 + i
+                    end_subword_index = start_subword_index
+                    break
+    return subword_to_word_mapping
+
+
+def store_validation_data_wiki(checkpoints_root, batch_size, label_size, is_training, use_retokenized_wikipedia_data):
+    dataset_name = f"validation_data_cache_b_{batch_size}_l_{label_size}_" \
+                   f"{('rt_wiki' if use_retokenized_wikipedia_data else 'wiki') if is_training else 'conll'}/"
+    if not os.path.exists(os.path.join(checkpoints_root, dataset_name)):
+        os.mkdir(os.path.join(checkpoints_root, dataset_name))
+    else:
+        print("Retrieving the validation data ...")
+        return dataset_name
+    print("Caching the validation data ...")
+    if is_training:
+        valid_iter = tqdm(get_dataset(
+            dataset_name='enwiki', split='valid', batch_size=batch_size, label_size=label_size,
+            use_retokenized_wikipedia_data=use_retokenized_wikipedia_data))
+    else:
+        valid_iter = tqdm(get_dataset(dataset_name='aida', split='valid', batch_size=batch_size, label_size=label_size))
+    for ind, (inputs, subword_mentions) in enumerate(valid_iter):
+        with open(os.path.join(checkpoints_root, dataset_name, f"{ind}"), "wb") as store_file:
+            pickle.dump((inputs.token_ids.cpu(), subword_mentions.ids.cpu(), subword_mentions.probs.cpu(),
+                         inputs.eval_mask.cpu(), subword_mentions.dictionary, inputs.raw_mentions,
+                         inputs.is_in_mention.cpu(), inputs.bioes.cpu()), store_file,
+                        protocol=pickle.HIGHEST_PROTOCOL)
+    return dataset_name
+
+
+def postprocess_annotations(annotations, sentence):
+    res = []
+    for ann in annotations:
+        begin_index = ann[0]
+        end_index = ann[1]
+        annotation = ann[2]
+        requires_check = True
+        while requires_check and end_index > begin_index:
+            mention = sentence[begin_index:end_index]
+            if mention.lower().endswith("\'s") and all([any([m in c for c in annotation[0].lower().split("_")])
+                                                        for m in mention[:-2].lower().split()]) and not \
+                    all([any([m in c for c in annotation[0].lower().split("_")]) for m in mention.lower().split()]):
+                end_index -= 2
+            elif mention[0] in string.punctuation or mention[0] == ' ':
+                begin_index += 1
+            elif mention[-1] in string.punctuation and mention.lower()[-4:] not in ["u.s.", "u.n."]:
+                end_index -= 1
+            elif mention[-1] == ' ':
+                end_index -= 1
+            elif mention.lower()[-3:] in ["u.s", "u.n"] and end_index < len(sentence) and sentence[end_index] == '.':
+                end_index += 1
+            elif mention.lower() in ["a", "the", "in", "out", "to", "of", "for", "at", "by", "rd", "th", "and",
+                                     "or", "but", "on", "none", "is", "were", "was", "he", "she", "if", "as",
+                                     "have", "had", "has", "who", "when", "where", "a lot", "a little", "here",
+                                     "there", "\'s"]:
+                end_index = begin_index
+                requires_check = False
+            else:
+                requires_check = False
+        if begin_index < end_index:
+            res.append((begin_index, end_index, annotation))
+    return res
+
+
+def get_aida_set_phrase_splitted_documents(dataset_name):
+    d_iter = AIDADataset().dataset[dataset_name]
+
+    phrase_documents = []
+
+    for document in d_iter:
+        document_words = []
+        document_labels = []
+        document_candidates = []
+
+        for annotation in document.annotations:
+            for a in annotation:
+                document_words.append(a.token)
+                document_candidates.append([x.url.replace('http://en.wikipedia.org/wiki/', '')
+                                            for x in a.candidates.candidates] if a.candidates else [])
+                if a.yago_entity and a.yago_entity != "--NME--":
+                    document_labels.append(a.yago_entity.encode('ascii').decode('unicode-escape'))
+                else:
+                    document_labels.append('|||O|||')
+        original_string = " ".join(document_words)
+        tokenized_mention = tokenizer(original_string)
+        tokens_offsets = list(zip(tokenized_mention.tokens(), tokenized_mention.encodings[0].offsets))[1:-1]
+        mapping = get_subword_to_word_mapping(tokenized_mention.tokens(), original_string)
+        subword_tokens = tokenized_mention.tokens()[1:-1]
+        w_ind = 0
+        subword_annotations = []
+        word_annotations = []
+        for w, l, cnds in zip(document_words, document_labels, document_candidates):
+            converted_to_words = "".join([x[1:] if x.startswith("\u0120")
+                                          else x for x in subword_tokens[mapping[w_ind][0]:mapping[w_ind][1]]])
+            if w == converted_to_words:
+                for sub_w in subword_tokens[mapping[w_ind][0]:mapping[w_ind][1]]:
+                    subword_annotations.append(SubwordAnnotation([1.0], [dl_sa.mentions_vocab[l]], sub_w))
+                word_annotations.append(WordAnnotation(subword_annotations[mapping[w_ind][0]:mapping[w_ind][1]],
+                                                       tokens_offsets[mapping[w_ind][0]:mapping[w_ind][1]], cnds))
+                w_ind += 1
+            elif len(mapping) > w_ind + 1 and w == "".join([x[1:] if x.startswith("\u0120")
+                                                            else x for x in subword_tokens[
+                                                                            mapping[w_ind][0]:mapping[w_ind+1][1]]]):
+                for sub_w in subword_tokens[mapping[w_ind][0]:mapping[w_ind+1][1]]:
+                    subword_annotations.append(SubwordAnnotation([1.0], [dl_sa.mentions_vocab[l]], sub_w))
+                word_annotations.append(WordAnnotation(subword_annotations[mapping[w_ind][0]:mapping[w_ind+1][1]],
+                                                       tokens_offsets[mapping[w_ind][0]:mapping[w_ind+1][1]], cnds))
+                w_ind += 2
+            else:
+                raise ValueError("This should not happen")
+        phrase_annotations = []
+        for w in word_annotations:
+            if phrase_annotations and phrase_annotations[-1].resolved_annotation == w.resolved_annotation:
+                phrase_annotations[-1].add(w)
+            else:
+                phrase_annotations.append(PhraseAnnotation(w))
+        phrase_documents.append(phrase_annotations)
+    return phrase_documents
+
+
+def _process_last_overlap(text_chunk_overlap, _overlap, l):
+    """
+    Function intended to merge the predictions in the text chunk overlaps.
+    Implemented to be used in chunk_annotate_and_merge_to_phrase function.
+    """
+    if not l:
+        l = _overlap
+    if len(l) < len(_overlap):
+        o = [x for x in _overlap]
+        o[-len(l):] = l
+        l = o
+    _r = []
+    if len(_overlap) < text_chunk_overlap:
+        text_chunk_overlap = len(_overlap)
+    for i in range(text_chunk_overlap):
+        if _overlap[i] == 0:
+            _r.append((l[i],))
+        elif l[i] == 0 or _overlap[i] == l[i]:
+            _r.append((_overlap[i],))
+        else:  # keeping both for prediction resolution
+            _r.append((l[i], _overlap[i]))
+    return _r
+
+
+def normalize_sentence_for_moses_alignment(sentence, normalize_for_chinese_characters=False):
+    for k, v in [('\u2018', '\''), ('\u2019', '\''), ('\u201d', '\"'), ('\u201c', '\"'), ('\u2013', '-'),
+                 ('\u2014', '-'), ('\u2026', '.'), ('\u2022', '.'), ('\u00f6', 'o'),('\u00e1', 'a'), ('\u00e8', 'e'),
+                 ('\u00c9', 'E'), ('\u014d', 'o'), ('\u0219', 's'), ('\n', '\u010a'), ('\u00a0', ' '), ('\u694a', ' '),
+                 ('\u9234', ' '), ('\u6797', ' '), ('\u6636', ' '), ('\u4f50', ' '), ('\u738b', ' '), ('\u5b9c', ' '),
+                 ('\u6b63', ' '), ('\u5168', ' '), ('\u52dd', ' '), ('\u80e1', ' '), ('\u5fd7', ' '), ('\u535a', ' '),
+                 ('\u9673', ' '), ('\u7f8e', ' '), ('\u20ac', 'E'), ('\u201e', '\"'), ('\u0107', 'c'), ('\ufeff', ' '),
+                 ('\u017e', 'z'), ('\u010d', 'c')]:
+        if k in sentence:
+            sentence = sentence.replace(k, v)
+    if normalize_for_chinese_characters:
+        for k, v in [('\u5e7c', ' '), ('\u5049', ' '), ('\u5b8f', ' '), ('\u9054', ' '), ('\u5bb9', ' '),
+                     ('\u96fb', ' '), ('\u590f', ' '), ('\u5b63', ' '), ('\u660c', ' '), ('\u90b1', ' '),
+                     ('\u4fca', ' '), ('\u6587', ' '), ('\u56b4', ' '), ('\u5b87', ' '), ('\u67cf', ' '),
+                     ('\u8b5a', ' '), ('\u9f0e', ' '), ('\u6176', ' '), ('\u99ac', ' '), ('\u82f1', ' '),
+                     ('\u4e5d', ' '), ('\u6797', ' '), ('\u7537', ' '), ('\u9996', ' '), ('\u60e0', ' '),
+                     ('\u7d00', ' '), ('\u5143', ' '), ('\u8f1d', ' '), ('\u5289', ' '), ('\u4fd0', ' '),
+                     ('\u8208', ' '), ('\u4e2d', ' '), ('\u8b1d', ' '), ('\u5922', ' '), ('\u9e9f', ' '),
+                     ('\u6e38', ' '), ('\u570b', ' '), ('\u7167', ' '), ('\u658c', ' '), ('\u54f2', ' '),
+                     ('\u9ec3', ' '), ('\u5433', ' '), ('\u53cb', ' '), ('\u6e05', ' '), ('\u856d', ' '),
+                     ('\u8000', ' '), ('\u5eb7', ' '), ('\u6dd1', ' '), ('\u83ef', ' ')]:
+            if k in sentence:
+                sentence = sentence.replace(k, v)
+    return sentence
+
+
+def chunk_annotate_and_merge_to_phrase(model, sentence, k_for_top_k_to_keep=5, normalize_for_chinese_characters=False):
+    sentence = sentence.rstrip()
+    sentence = normalize_sentence_for_moses_alignment(sentence, normalize_for_chinese_characters)
+    simple_split_words = moses_tokenize(sentence)
+    sentence = sentence.replace('\u010a', '\n')
+    tokenized_mention = tokenizer(sentence)
+    tokens_offsets = list(zip(tokenized_mention.tokens(), tokenized_mention.encodings[0].offsets))
+    subword_to_word_mapping = get_subword_to_word_mapping(tokenized_mention.tokens(), sentence)
+    chunks = [tokens_offsets[i: i + model.text_chunk_length] for i in range(
+        0, len(tokens_offsets), model.text_chunk_length - model.text_chunk_overlap)]
+    result = []
+    last_overlap = []
+    logits = []
+    # ########################################################################################################
+    # Covert each chunk to tensors, predict the labels, and merge the overlaps (keep conflicting predictions).
+    # ########################################################################################################
+    for chunk in chunks:
+        subword_ids = [tokenizer.convert_tokens_to_ids([x[0] for x in chunk])]
+        logits = model.annotate_subword_ids(
+            subword_ids, k_for_top_k_to_keep, chunk)
+        if last_overlap:
+            result.extend(_process_last_overlap(model.text_chunk_overlap, last_overlap, logits))
+        else:
+            result.extend([(x,) for x in logits[:model.text_chunk_overlap]])
+        if len(logits) > 2 * model.text_chunk_overlap:
+            result.extend([(x,) for x in logits[model.text_chunk_overlap:-model.text_chunk_overlap]])
+            last_overlap = logits[-model.text_chunk_overlap:]
+        else:
+            result.extend([(x,) for x in logits[model.text_chunk_overlap:]])
+            last_overlap = []
+        logits = []
+    result.extend(_process_last_overlap(model.text_chunk_overlap, last_overlap, logits))
+    # ########################################################################################################
+    # Resolve the overlap merge conflicts using the model prediction probability
+    # ########################################################################################################
+    final_result = []
+    for p_ind, prediction in enumerate(result):
+        if len(prediction) == 1:
+            final_result.append(prediction[0])
+        else:
+            p_found = False
+            for p in prediction:
+                if p == final_result[-1] or (p_ind + 1 < len(result) and p in result[p_ind + 1]):
+                    # It is equal to the one in the left or in the one to the right
+                    final_result.append(p)
+                    p_found = True
+                    break
+            if not p_found:  # choose the one the model is more confident about
+                final_result.append(sorted(prediction, key=lambda x: x.item_probability(), reverse=True)[0])
+    # ########################################################################################################
+    # Convert the model predictions (subword-level) to valid GERBIL annotation spans (continuous char-level)
+    # ########################################################################################################
+    tokens_offsets = tokens_offsets[1:-1]
+    final_result = final_result[1:]
+    # last_step_annotations = []
+    word_annotations = [WordAnnotation(final_result[m[0]:m[1]], tokens_offsets[m[0]:m[1]])
+                        for m in subword_to_word_mapping]
+    # ########################################################################################################
+    #                    MAKING SURE WORDS ARE NOT BROKEN IN SEPARATE PHRASES!
+    # ########################################################################################################
+    w_p_1 = 0
+    w_p_2 = 0
+    w_2_buffer = ""
+    w_1_buffer = ""
+    while w_p_1 < len(word_annotations) and w_p_2 < len(simple_split_words):
+        w_1 = word_annotations[w_p_1]
+        w_2 = normalize(simple_split_words[w_p_2]).strip()
+        w_1_word_string = normalize(w_1.word_string).strip()
+        if w_1_word_string == w_2:
+            w_p_1 += 1
+            w_p_2 += 1
+        elif w_1_buffer and w_2_buffer and normalize(
+                w_1_buffer + w_1.word_string).strip() == normalize(w_2_buffer + simple_split_words[w_p_2]).strip():
+            w_p_1 += 1
+            w_p_2 += 1
+            w_1_buffer = ""
+            w_2_buffer = ""
+        elif w_2_buffer and w_1_word_string == normalize(w_2_buffer + simple_split_words[w_p_2]).strip():
+            w_p_1 += 1
+            w_p_2 += 1
+            w_2_buffer = ""
+        elif w_1_buffer and normalize(w_1_buffer + w_1.word_string).strip() == w_2:
+            w_p_1 += 1
+            w_p_2 += 1
+            w_1_buffer = ""
+        elif w_1_buffer and len(w_2) < len(normalize(w_1_buffer + w_1.word_string).strip()):
+            w_2_buffer += simple_split_words[w_p_2]
+            w_p_2 += 1
+        elif len(w_2) < len(w_1_word_string):
+            w_2_buffer += simple_split_words[w_p_2]
+            w_p_2 += 1
+        # Connecting the "." in between the names to the word it belongs to.
+        elif len(w_2) > len(w_1_word_string) and w_p_1 + 1 < len(word_annotations) \
+                and word_annotations[w_p_1 + 1].word_string == ".":
+            word_annotations[w_p_1 + 1] = WordAnnotation(
+                word_annotations[w_p_1].annotations + word_annotations[w_p_1 + 1].annotations,
+                word_annotations[w_p_1].token_offsets + word_annotations[w_p_1 + 1].token_offsets)
+            word_annotations[w_p_1].annotations = []
+            word_annotations[w_p_1].token_offsets = []
+            w_p_1 += 1
+        elif len(w_2) > len(w_1_word_string) and w_p_1 + 1 < len(word_annotations):
+            w_1_buffer += w_1.word_string
+            w_p_1 += 1
+        elif w_2_buffer and normalize(word_annotations[w_p_1].word_string + word_annotations[w_p_1 + 1].word_string).strip():
+            w_p_1 += 2
+            w_2_buffer = ""
+        else:
+            raise ValueError("This should not happen!")
+    # ################################################################################################################
+    phrase_annotations = []
+    for w in word_annotations:
+        if not w.annotations:
+            continue
+        if phrase_annotations and phrase_annotations[-1].resolved_annotation == w.resolved_annotation:
+            phrase_annotations[-1].add(w)
+        else:
+            phrase_annotations.append(PhraseAnnotation(w))
+    return phrase_annotations
+
+
+class ComparisonResult(Enum):
+    CORRECTLY_IGNORED_O = 0
+    CORRECTLY_FOUND_BOTH_SPAN_AND_ANNOTATION = 1
+    CORRECTLY_FOUND_SPAN_BUT_NOT_ANNOTATION = 2
+    OVER_GENERATED_ANNOTATION = 3
+
+    @staticmethod
+    def get_correct_status(g_span, p_span):
+        g_is_o = g_span.resolved_annotation == 0
+        got_annotation_right = p_span.resolved_annotation == g_span.resolved_annotation
+        got_span_right = p_span.word_string.replace(" ", "") == g_span.word_string.replace(" ", "")
+        #  p_span.average_annotation_confidence == g_span.average_annotation_confidence
+        if got_span_right and got_annotation_right and g_is_o:
+            return ComparisonResult.CORRECTLY_IGNORED_O
+        elif got_span_right and got_annotation_right and not g_is_o:
+            return ComparisonResult.CORRECTLY_FOUND_BOTH_SPAN_AND_ANNOTATION
+        elif got_span_right and not got_annotation_right and not g_is_o:
+            # it could be that p is o or not!
+            return ComparisonResult.CORRECTLY_FOUND_SPAN_BUT_NOT_ANNOTATION
+        elif got_span_right and not got_annotation_right and g_is_o:
+            return ComparisonResult.OVER_GENERATED_ANNOTATION
+        else:
+            raise ValueError("This should not happen!")
+
+
+def compare_gold_and_predicted_annotation_documents(gold_document, predicted_document, ignore_over_generated=False,
+                                                    ignore_predictions_outside_candidate_list=False):
+    """
+    Compares the output results of the model predictions and the gold annotations.
+    """
+    g_id = 0
+    p_id = 0
+    comparison_results = []
+    while g_id < len(gold_document) and p_id < len(predicted_document):
+        p_span = predicted_document[p_id]
+        g_span = gold_document[g_id]
+        special_condition = p_span.word_string != g_span.word_string and p_span.word_string.replace(
+            " ", "") == g_span.word_string.replace(" ", "")
+        if p_span.word_string == g_span.word_string or special_condition:
+            p_id += 1
+            g_id += 1
+            comparison_results.append((g_span, p_span, ComparisonResult.get_correct_status(g_span, p_span)))
+        elif len(p_span.word_string) < len(g_span.word_string) and \
+                len(p_span.words) == len(g_span.words) == 1 and p_id + 1 < len(predicted_document) and \
+                len(predicted_document[p_id+1].words) > 1:
+            p_span.add(predicted_document[p_id+1].words[0])
+            predicted_document[p_id+1].words.pop(0)
+            continue
+        elif len(p_span.word_string) < len(g_span.word_string):
+            # potentially over-generated span later
+            new_phrase = PhraseAnnotation(g_span.words[0])
+            i = 1
+            while new_phrase.word_string.replace(" ", "") != p_span.word_string.replace(" ", "") \
+                    and i < len(g_span.words):
+                new_phrase.add(g_span.words[i])
+                i += 1
+            not_solved = new_phrase.word_string.replace(" ", "") != p_span.word_string.replace(" ", "")
+            if not_solved and p_id + 1 < len(predicted_document) and len(predicted_document[p_id+1].words) > 1:
+                p_span.add(predicted_document[p_id+1].words[0])
+                predicted_document[p_id+1].words.pop(0)
+                continue
+            elif not_solved and p_id + 1 < len(predicted_document) and len(predicted_document[p_id+1].words) == 1:
+                p_span.add(predicted_document[p_id+1].words[0])
+                predicted_document[p_id+1].words = p_span.words
+                predicted_document[p_id+1].set_alternative_as_resolved_annotation(p_span.resolved_annotation)
+                p_id += 1
+                continue
+            elif not_solved:
+                raise ValueError("This should not happen!")
+            else:
+                comparison_results.append((
+                    new_phrase, p_span, ComparisonResult.get_correct_status(new_phrase, p_span)))
+                g_span.words = g_span.words[i:]
+                p_id += 1
+        elif len(p_span.word_string) > len(g_span.word_string):
+            # potentially missed a span
+            new_phrase = PhraseAnnotation(p_span.words[0])
+            i = 1
+            while new_phrase.word_string.replace(" ", "") != g_span.word_string.replace(" ", "") \
+                    and i < len(p_span.words):
+                new_phrase.add(p_span.words[i])
+                i += 1
+            if new_phrase.word_string.replace(" ", "") != g_span.word_string.replace(" ", ""):
+                # re-alignment not helpful
+                new_p = PhraseAnnotation(p_span.words[0])
+                new_g = PhraseAnnotation(g_span.words[0])
+                i = 1
+                while new_p.word_string == new_g.word_string:
+                    new_p.add(p_span.words[i])
+                    new_g.add(g_span.words[i])
+                    i += 1
+                new_p.words = new_p.words[:-1]
+                new_g.words = new_g.words[:-1]
+                comparison_results.append((new_g, new_p, ComparisonResult.get_correct_status(new_g, new_p)))
+                p_span.words = p_span.words[i - 1:]
+                g_span.words = g_span.words[i - 1:]
+            else:
+                comparison_results.append((
+                    g_span, new_phrase, ComparisonResult.get_correct_status(g_span, new_phrase)))
+                p_span.words = p_span.words[i:]
+                g_id += 1
+        elif g_span.word_string.replace(" ", "").startswith(p_span.word_string.replace(" ", "")) and \
+                p_id + 1 < len(predicted_document) and p_span.word_string.replace(" ", "") + \
+                predicted_document[p_id + 1].word_string.replace(" ", "") == g_span.word_string.replace(" ", ""):
+            for next_span_word in predicted_document[p_id+1].words:
+                p_span.add(next_span_word)
+            predicted_document[p_id+1] = p_span
+            p_id += 1
+            continue
+        elif g_span.word_string.replace(" ", "").startswith(p_span.word_string.replace(" ", "")) and \
+                p_id + 1 < len(predicted_document) and p_span.word_string.replace(" ", "") + \
+                predicted_document[p_id + 1].words[0].word_string.replace(" ", "") == \
+                g_span.word_string.replace(" ", ""):
+            p_span.add(predicted_document[p_id+1].words[0])
+            predicted_document[p_id+1].words.pop(0)
+            continue
+        elif g_span.word_string.replace(" ", "").startswith(p_span.word_string.replace(" ", "")):
+            raise ValueError("This should be handled correctly!")
+        elif p_span.word_string.replace(" ", "").startswith(g_span.word_string.replace(" ", "")):
+            raise ValueError("This should be handled correctly!")
+        else:
+            raise ValueError("This should not happen!")
+    if ignore_over_generated:
+        c_results = []
+        for g, p, r in comparison_results:
+            if ignore_over_generated and r == ComparisonResult.OVER_GENERATED_ANNOTATION:
+                p.set_alternative_as_resolved_annotation(0)
+                r = ComparisonResult.CORRECTLY_IGNORED_O
+            c_results.append((g, p, r))
+        comparison_results = c_results
+    if ignore_predictions_outside_candidate_list:
+        c_results = []
+        for g, p, r in comparison_results:
+            g_ppr_for_ned_candidates = [dl_sa.mentions_vocab[x] for x in g.ppr_for_ned_candidates if x in dl_sa.mentions_vocab]
+            if g_ppr_for_ned_candidates:
+                all_p_anns = p.all_possible_annotations()
+                filtered_p_predictions = sorted([x for x in all_p_anns if x[0] in g_ppr_for_ned_candidates],
+                                                key=lambda y: y[1], reverse=True)
+                if filtered_p_predictions:
+                    p.set_alternative_as_resolved_annotation(filtered_p_predictions[0][0])
+                else:
+                    p.set_alternative_as_resolved_annotation(0)
+                r = ComparisonResult.get_correct_status(g, p)
+            c_results.append((g, p, r))
+        comparison_results = c_results
+    return comparison_results