multi model update

app.py

@@ -52,15 +52,26 @@ def load_model_and_tokenizer():
     print(type(tokenizer))
     print(type(model))
     return model, tokenizer
+    
+@st.cache(hash_funcs={transformers.models.bert.tokenization_bert_fast.BertTokenizerFast: lambda _: None, transformers.models.bert.modeling_bert.BertModel: lambda _: None})
+def load_hu_model_and_tokenizer():
+    multilingual_checkpoint = 'sentence-transformers/SZTAKI-HLT/hubert-base-cc'  #alternative: SZTAKI-HLT/hubert-base-cc 
+    tokenizer = AutoTokenizer.from_pretrained(multilingual_checkpoint)
+    model = AutoModel.from_pretrained(multilingual_checkpoint)
+    print(type(tokenizer))
+    print(type(model))
+    return model, tokenizer
 
 
 model,tokenizer = load_model_and_tokenizer();
+model_hu,tokenizer_hu = load_hu_model_and_tokenizer();
 raw_text_file = 'joint_text_filtered.md'
 all_sentences = load_raw_sentences(raw_text_file)
 
 embeddings_file = 'multibert_embedded.pt' #alternative: hunbert_embedded.pt
 all_embeddings = load_embeddings(embeddings_file)
-
+embeddings_file_hu = 'hunbert_embedded.pt'
+all_embeddings_hu = load_embeddings(embeddings_file_hu)
 
 st.header('RF szöveg kereső')
 
@@ -74,6 +85,9 @@ if text_area_input_query:
     query_embedding = calculateEmbeddings([text_area_input_query],tokenizer,model)
     top_pairs = findTopKMostSimilar(query_embedding, all_embeddings, all_sentences, 5)
     st.json(top_pairs)
+    query_embedding = calculateEmbeddings([text_area_input_query],tokenizer_hu,model_hu)
+    top_pairs = findTopKMostSimilar(query_embedding, all_embeddings_hu, all_sentences, 5)
+    st.json(top_pairs)