in progress, need search engine

app.py

@@ -0,0 +1,53 @@
+import re 
+import streamlit as st
+import textract
+import tempfile
+import random
+from pathlib import Path
+import spacy
+from spacy.tokens import DocBin
+import srsly
+from spacy.matcher import Matcher
+
+st.title('Index and Search a Collection of Documents')
+
+@st.cache
+def download_model(select_model:str):
+    with st.spinner(f'Loading model {select_model}'):
+        spacy.cli.download(select_model)
+    return True
+
+doc_bin = DocBin()
+models = srsly.read_json('models.json')
+models[''] = [] #require the user to choose a language
+languages = models.keys()
+language = st.selectbox("Language", languages, index=len(models.keys())-1, help="Select the language of your materials.")
+if language:
+    select_model = st.selectbox("Model", models[language], help="spaCy model")
+    if select_model:
+        model_downloaded = download_model(select_model)
+
+        if model_downloaded:
+
+            nlp = spacy.load(select_model)
+
+            nlp.max_length = 1200000
+
+            uploaded_files = st.file_uploader("Select files to process", accept_multiple_files=True)
+
+            for uploaded_file in uploaded_files:
+                file_type = uploaded_file.type
+
+                temp = tempfile.NamedTemporaryFile()
+                temp.write(uploaded_file.getvalue())
+                try:
+                    text = textract.process(temp.name)
+                    text = text.decode('utf-8')
+                    doc = nlp(text)
+                    st.write(text)
+                except Exception as e:
+                    st.error(e)
+                
+
+            #st.download_button('Download', '', 'text/plain')
+

models.json

@@ -0,0 +1,149 @@
+{
+    "Catalan": [
+        "",
+        "ca_core_news_sm",
+        "ca_core_news_md",
+        "ca_core_news_lg",
+        "ca_core_news_trf"
+    ],
+    "Chinese":[
+        "",
+        "zh_core_web_sm",
+        "zh_core_web_md",
+        "zh_core_web_lg",
+        "zh_core_web_trf"
+    ],
+    "Croatian": [
+        "",
+        "hr_core_news_sm",
+        "hr_core_news_md",
+        "hr_core_news_lg"
+    ],
+    "Danish": [
+        "",
+        "da_core_news_sm",
+        "da_core_news_md",
+        "da_core_news_lg",
+        "da_core_news_trf"
+    ],
+    "Dutch":[
+        "",
+        "nl_core_news_sm",
+        "nl_core_news_md",
+        "nl_core_news_lg"
+    ],
+    "English": [
+        "",
+        "en_core_web_sm",
+        "en_core_web_md",
+        "en_core_web_lg",
+        "en_core_web_trf"
+    ],
+    "Finnish":[
+        "",
+        "fi_core_news_sm",
+        "fi_core_news_md",
+        "fi_core_news_lg"
+    ],
+    "French":[
+        "",
+        "fr_core_news_sm",
+        "fr_core_news_md",
+        "fr_core_news_lg",
+        "fr_dep_news_trf"
+    ],
+    "German": [
+        "",
+        "de_core_news_sm",
+        "de_core_news_md",
+        "de_core_news_lg",
+        "de_dep_news_trf"
+    ],
+    "Greek":[
+        "",
+        "el_core_news_sm",
+        "el_core_news_md",
+        "el_core_news_lg"
+    ],
+    "Italian":[
+        "",
+        "it_core_news_sm",
+        "it_core_news_md",
+        "it_core_news_lg"
+    ],
+    "Japanese":[
+        "",
+        "ja_core_news_sm",
+        "ja_core_news_md",
+        "ja_core_news_lg",
+        "ja_core_news_trf"
+    ],
+    "Korean":[
+        "",
+        "ko_core_news_sm",
+        "ko_core_news_md",
+        "ko_core_news_lg"
+    ],
+    "Lithuanian":[
+        "",
+        "lt_core_news_sm",
+        "lt_core_news_md",
+        "lt_core_news_lg"
+    ],
+    "Macedonian":[
+        "",
+        "mk_core_news_sm",
+        "mk_core_news_md",
+        "mk_core_news_lg"
+    ],
+    "Multi-language":[
+        "",
+        "xx_ent_wiki_sm",
+        "xx_sent_ud_sm"
+    ],
+    "Norwegian Bokmål":[
+        "",
+        "nb_core_news_sm",
+        "nb_core_news_md",
+        "nb_core_news_lg"
+    ],
+    "Polish":[
+        "",
+        "pl_core_news_sm",
+        "pl_core_news_md",
+        "pl_core_news_lg"
+    ],
+    "Portuguese":[
+        "",
+        "pt_core_news_sm",
+        "pt_core_news_md",
+        "pt_core_news_lg"
+    ],
+    "Romanian":[
+        "",
+        "ro_core_news_sm",
+        "ro_core_news_md",
+        "ro_core_news_lg"
+    ],
+    "Russian":[
+        "",
+        "ru_core_news_sm",
+        "ru_core_news_md",
+        "ru_core_news_lg"
+    ],
+    "Spanish":[
+        "",
+        "es_core_news_sm",
+        "es_core_news_md",
+        "es_core_news_lg",
+        "es_dep_news_trf"
+    ],
+    "Swedish":[
+        "",
+        "sv_core_news_sm",
+        "sv_core_news_md",
+        "sv_core_news_lg"
+        
+    ]     
+
+}

packages.txt

@@ -0,0 +1,15 @@
+python-dev
+libxml2-dev
+libxslt1-dev
+antiword
+unrtf
+poppler-utils
+tesseract-ocr
+flac
+ffmpeg
+lame
+libmad0
+libsox-fmt-mp3
+sox
+libjpeg-dev
+swig

requirements.txt

@@ -0,0 +1,4 @@
+streamlit==1.11.1
+textract==1.6.5
+spacy==3.4.1
+#spacy-streamlit==1.0.4