app.py

import gradio as gr
import languagecodes
import inspect
import httpx, os
import polars as pl 

df = pl.read_parquet("isolanguages.parquet")
non_empty_isos = df.slice(1).filter(pl.col("ISO639-1") != "").rows()
# all_langs = languagecodes.iso_languages_byname
all_langs = {iso[0]: (iso[1], iso[2], iso[3]) for iso in non_empty_isos} # {'Romanian': ('ro', 'rum', 'ron')}
iso1toall = {iso[1]: (iso[0], iso[2], iso[3]) for iso in non_empty_isos} # {'ro': ('Romanian', 'rum', 'ron')}
DEFAULTS = None
  
libraries = ["langdetect", "py3langid", "langid", "lingua-py", "pycld2", "fastlangdetect", "fasttext", "openlid", "glotlid"]

class Detect():
    def __init__(self, text: str) -> None:
        self.text: str = text
    def langdetect(self) -> list[str, float]:
        from langdetect import detect, detect_langs
        from langdetect import DetectorFactory
        DetectorFactory.seed = 0
        langcode = detect(self.text)
        langecode_probabilities: list[Language] = detect_langs(self.text)
        return [langcode, round(number=langecode_probabilities[0].prob * 100, ndigits=2)]
    def langid(self) -> list[str, float]:
        from langid.langid import LanguageIdentifier, model
        identifier = LanguageIdentifier.from_modelstring(string=model, norm_probs=True)
        idresult: list[str, float] = list(identifier.classify(self.text))
        return [idresult[0], abs(round(number=idresult[1] * 100, ndigits=2))]
    def py3langid(self) -> list[str, float]:
        langs = ["af", "am", "an", "ar", "as", "az", "be", "bg", "bn", "br", "bs", "ca", "cs", "cy",
                 "da", "de", "dz", "el", "en", "eo", "es", "et", "eu", "fa", "fi", "fo", "fr", "ga",
                 "gl", "gu", "he", "hi", "hr", "ht", "hu", "hy", "id", "is", "it", "ja", "jv", "ka",
                 "kk", "km", "kn", "ko", "ku", "ky", "la", "lb", "lo", "lt", "lv", "mg", "mk", "ml",
                 "mn", "mr", "ms", "mt", "nb", "ne", "nl", "nn", "no", "oc", "or", "pa", "pl", "ps",
                 "pt", "qu", "ro", "ru", "rw", "se", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta",
                 "te", "th", "tl", "tr", "ug", "uk", "ur", "vi", "vo", "wa", "xh", "zh", "zu"]
        import py3langid
        lang, prob = py3langid.classify(self.text) # unpack the result tuple in variables
        return [lang, abs(round(number=prob, ndigits=2))]       
    def lingua(self) -> list[str, float]:
        from lingua import Language, LanguageDetectorBuilder
        detector: LanguageDetector = LanguageDetectorBuilder.from_all_languages().with_preloaded_language_models().build()
        confidence_values: List[ConfidenceValue] = detector.compute_language_confidence_values(self.text)
        return [confidence_values[0].language.iso_code_639_1.name.lower(), "{0:.2f}".format(confidence_values[0].value * 100)]
    def fasttextlangdetect(self) -> list[str, float]: # requires numpy < 2.0
        from ftlangdetect import detect
        result = detect(text=self.text, low_memory=False)
        return [result.get('lang'), abs(round(number=result.get('score') * 100, ndigits=2))]
    def fastlangdetect(self) -> list[str, float]:
        from fast_langdetect import detect
        result = detect(text=self.text, model="auto", k=1)[0]
        return [result.get('lang'), abs(round(number=result.get('score') * 100, ndigits=2))]
    def pycld2(self) -> list[str, float]:
        import pycld2 as cld2
        # available_languages = cld2.LANGUAGES
        isReliable, textBytesFound, details = cld2.detect(self.text, returnVectors=False, bestEffort=True)
        return [details[0][1], round(details[0][2], 2)]
    def parse_fastext(self, repo_id, k=3):
        import fasttext
        from huggingface_hub import hf_hub_download
        model_path = hf_hub_download(repo_id=repo_id, filename="model.bin")
        model = fasttext.load_model(model_path)
        language, probabilities = model.predict(self.text, k=k)
        reversed_nllb_langs = {v: k for k, v in languagecodes.nllb_language_codes.items()}
        long_langname = reversed_nllb_langs[language[0].replace('__label__', '')]
        lang_code = all_langs[long_langname][0]
        return [lang_code, round(number=probabilities[0] * 100, ndigits=2)]     
    def fasttext(self) -> list[str, float]:
        import fasttext
        from huggingface_hub import hf_hub_download
        model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification", filename="model.bin")
        model = fasttext.load_model(model_path)
        language, probabilities = model.predict(self.text, k=3)
        reversed_nllb_langs = {v: k for k, v in languagecodes.nllb_language_codes.items()}
        long_langname = reversed_nllb_langs[language[0].replace('__label__', '')]
        lang_code = all_langs[long_langname][0]
        return [lang_code, round(number=probabilities[0] * 100, ndigits=2)]
    def openlid(self) -> list[str, float]:
        import fasttext
        from huggingface_hub import hf_hub_download
        model_path = hf_hub_download(repo_id="laurievb/OpenLID-v2", filename="model.bin")
        model = fasttext.load_model(model_path)
        language, probabilities = model.predict(self.text, k=3)
        reversed_nllb_langs = {v: k for k, v in languagecodes.nllb_language_codes.items()}
        long_langname = reversed_nllb_langs[language[0].replace('__label__', '')]
        lang_code = all_langs[long_langname][0]
        return [lang_code, round(number=probabilities[0] * 100, ndigits=2)]
    def glotlid(self) -> list[str, float]:
        repo_id="cis-lmu/glotlid"
        return self.parse_fastext(repo_id)

def detect_language(input_text: str, used_libraries: list[str]) -> tuple[str, str]:
    """
    Detects the language of the input text.

    Parameters:
        input_text (str): The source text to be translated
        used_libraries: (list[str]) The libraries to be used for detection
    Returns:
        list of lists with: 
            detected_text(str): The language code of the input text
            confidence(float):  The confidence score as float
    
    Example:
        >>> detect_language("Hello world", ["langdetect", "langid", "lingua-py", "fasttextlangdetect", "fastlangdetect"])
        [["en", 1.0]]
    """
    detectinstance = Detect(input_text)
    detections = []
    if not input_text or not used_libraries:
        return [['No input text or library selected', 'Please provide input text and/or select a detection library']]
    if 'langdetect' in used_libraries:
        detections.append(['langdetect'] + detectinstance.langdetect())
    if 'langid' in used_libraries:
        detections.append(['langid'] + detectinstance.langid())
    if 'py3langid' in used_libraries:
        detections.append(['py3langid'] + detectinstance.py3langid())
    if 'lingua-py' in used_libraries:
        detections.append(['lingua-py'] + detectinstance.lingua())
    if 'pycld2' in used_libraries:
        detections.append(['pycld2'] + detectinstance.pycld2())
    if 'fastlangdetect' in used_libraries:
        detections.append(['fastlangdetect'] + detectinstance.fastlangdetect())
    if 'fasttext' in used_libraries:
        detections.append(['fasttext'] + detectinstance.fasttext())
    if 'openlid' in used_libraries:
        detections.append(['openlid'] + detectinstance.openlid())
    if 'glotlid' in used_libraries:
        detections.append(['glotlid'] + detectinstance.glotlid())
    unique_codes = list(set([x[1] for x in detections]))
    unique_languages = [iso1toall[x][0] for x in unique_codes]
    detections.append([f'Unique languages: {unique_languages}', f'Unique codes: {unique_codes}', f'Languages detected: {len(unique_codes)}'])
    print(unique_codes, unique_languages, detections)
    return detections
    
with gr.Blocks() as interface:
    gr.Markdown("### Language Detection with Gradio API and MCP Server")
    input_text = gr.Textbox(label="Enter text to detect:", placeholder="Type/copy text here, maximum 512 characters",
                            autofocus=True, submit_btn='Detect Language', max_length=512) 
    with gr.Row(variant="compact"):
        used_libraries = gr.CheckboxGroup(choices=libraries, value=libraries, label="Detection libraries", show_select_all=True)
    dataframe = gr.Dataframe(
            headers=["Library", "Language  code", "Score"],
            datatype=["str", "str", "number"],
            type='array',
            row_count=len(libraries),
            column_count=3,
            column_limits=(2, 4),
            label='Language detection dataframe'
        )
    input_text.submit(
    fn=detect_language,
    inputs=[input_text, used_libraries],
    outputs=[dataframe]
    )
    
if __name__ == "__main__":
    interface.launch(mcp_server=True, footer_links=["api", "settings"])
    # interface.queue().launch(server_name="0.0.0.0", show_error=True, server_port=7860, mcp_server=True)