change scispacy versions

ner_datasets/preprocessing/preprocessing_loader.py

@@ -20,6 +20,8 @@ class PreprocessingLoader(object):
         """
         if sentencizer == 'en_core_sci_lg':
             return SpacySentencizer(spacy_model='en_core_sci_lg')
+        elif sentencizer == 'en_core_sci_sm':
+            return SpacySentencizer(spacy_model='en_core_sci_sm')
         elif sentencizer == 'en_core_web_sm':
             return SpacySentencizer(spacy_model='en_core_web_sm')
         elif sentencizer == 'note':
@@ -55,9 +57,8 @@ class PreprocessingLoader(object):
         elif tokenizer == 'clinical':
             # Abbreviations - we won't split tokens that match these (e.g 18F-FDG)
             if abbreviations is None:
-                return ClinicalSpacyTokenizer(spacy_model='en_core_sci_lg', abbreviations=abbreviations)
+                return ClinicalSpacyTokenizer(spacy_model='en_core_sci_sm', abbreviations=abbreviations)
             else:
-
-                return ClinicalSpacyTokenizer(spacy_model='en_core_sci_lg', abbreviations=abbreviations)
+                return ClinicalSpacyTokenizer(spacy_model='en_core_sci_sm', abbreviations=abbreviations)
         else:
             raise ValueError('Invalid tokenizer - does not exist')

requirements.txt

@@ -19,4 +19,5 @@ scispacy
 datasets
 pytorch-crf
 allennlp
-pycorenlp
+pycorenlp
+https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_sm-0.4.0.tar.gz