Added openlid

app.py

@@ -11,7 +11,7 @@ all_langs = {iso[0]: (iso[1], iso[2], iso[3]) for iso in non_empty_isos} # {'Rom
 iso1toall = {iso[1]: (iso[0], iso[2], iso[3]) for iso in non_empty_isos} # {'ro': ('Romanian', 'rum', 'ron')}
 DEFAULTS = None
   
-libraries = ["langdetect", "langid", "lingua-py", "pycld2", "fastlangdetect", "fasttext"]
+libraries = ["langdetect", "langid", "lingua-py", "pycld2", "fastlangdetect", "fasttext", "openlid"]
 
 class Detect():
     def __init__(self, text: str) -> None:
@@ -56,6 +56,16 @@ class Detect():
         long_langname = reversed_nllb_langs[language[0].replace('__label__', '')]
         lang_code = all_langs[long_langname][0]
         return [lang_code, round(number=probabilities[0] * 100, ndigits=2)]
+    def openlid(self) -> list[str, float]:
+        import fasttext
+        from huggingface_hub import hf_hub_download
+        model_path = hf_hub_download(repo_id="laurievb/OpenLID-v2", filename="model.bin")
+        model = fasttext.load_model(model_path)
+        language, probabilities = model.predict(self.text, , k=3)
+        reversed_nllb_langs = {v: k for k, v in languagecodes.nllb_language_codes.items()}
+        long_langname = reversed_nllb_langs[language[0].replace('__label__', '')]
+        lang_code = all_langs[long_langname][0]
+        return [lang_code, round(number=probabilities[0] * 100, ndigits=2)]
 
 def detect_language(input_text: str, used_libraries: list[str]) -> tuple[str, str]:
     """
@@ -89,6 +99,8 @@ def detect_language(input_text: str, used_libraries: list[str]) -> tuple[str, st
         detections.append(['fastlangdetect'] + detectinstance.fastlangdetect())
     if 'fasttext' in used_libraries:
         detections.append(['fasttext'] + detectinstance.fasttext())
+    if 'openlid' in used_libraries:
+        detections.append(['openlid'] + detectinstance.openlid())
     unique_languages = list(set([x[1] for x in detections]))
     print(unique_languages, detections)
     return detections