Fixed some code smell

app.py

@@ -8,12 +8,11 @@ import nltk, spacy, gensim
 from sklearn.decomposition import LatentDirichletAllocation
 from sklearn.feature_extraction.text import CountVectorizer
 from pprint import pprint
+import matplotlib
+matplotlib.use('agg')
 
-def concat_comments(sup_comment: list[str], comment: list[str]) -> list[str]:
-    format_s = "{s}\n{c}"
-    return [
-        format_s.format(s=s, c=c) for s, c in zip(sup_comment, comment)
-    ]
+def concat_comments(*kwargs):
+    return ['\n'.join(ele) for ele in zip(*kwargs)]
 
 def sent_to_words(sentences):
     for sentence in sentences:
@@ -28,38 +27,34 @@ def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'], nlp=Non
         ]))
     return texts_out
 
-
-def main(choose_context):
+def get_lda(n_components):
     df = pd.read_csv('./data/results.csv', index_col=0)
-
-    print(choose_context)
-
-    if choose_context == 'comment':
-        data = df.comment
-    elif choose_context == 'sup comment':
-        data = df.sup_comment
-    elif choose_context == 'sup comment + comment':
-        data = concat_comments(df.sup_comment, df.comment)
-
+    data = concat_comments(df.subreddit, df.sup_comment, df.comment)
     data_words = list(sent_to_words(data))
 
-    print('downloading en_core_web_sm')
-    os.system("python -m spacy download en_core_web_sm")
-    print('en_core_web_sm downloaded')
-
+    
+    if not spacy.util.is_package("en_core_web_sm"):
+        print('[x] en_core_web_sm not found, downloading...')
+        os.system("python -m spacy download en_core_web_sm")
+        print('[x] en_core_web_sm downloaded')
+    
+    print('[x] Lemmatization begins')
     nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])
     data_lemmatized = lemmatization(data_words, allowed_postags=["NOUN", "ADJ"], nlp=nlp) #select noun and verb
 
+    print('[x] Vectorizing')
     vectorizer = CountVectorizer(
-    analyzer='word',
-    min_df=10,
-    stop_words='english',
-    lowercase=True,
-    token_pattern='[a-zA-Z0-9]{3,}'
-)
-    data_vectorized = vectorizer.fit_transform(data_lemmatized)
+        analyzer='word',
+        min_df=10,
+        stop_words='english',
+        lowercase=True,
+        token_pattern='[a-zA-Z0-9]{3,}'
+    )
 
+    print('[x] Fitting vectorized data on lemmatization')
+    data_vectorized = vectorizer.fit_transform(data_lemmatized)
 
+    print('[x] Init LDA model')
     lda_model = LatentDirichletAllocation(
         n_components=5,
         max_iter=10,
@@ -68,19 +63,28 @@ def main(choose_context):
         batch_size=128,
         evaluate_every = -1,
         n_jobs = -1,
+        verbose=1,
     )
+    
+    print('[x] Fitting LDA model')
     lda_output = lda_model.fit_transform(data_vectorized)
     print(lda_model)    # Model attributes
 
+    print('[x] Getting performances')
+    performances = lda_model.score(data_vectorized), lda_model.perplexity(data_vectorized)
     # Log Likelyhood: Higher the better
-    print("Log Likelihood: ", lda_model.score(data_vectorized))
+    print("Log Likelihood: ", performances[0])
     # Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)
-    print("Perplexity: ", lda_model.perplexity(data_vectorized))
+    print("Perplexity: ",  performances[1])
+
+    print('[x] Check parameters if they look correct')
     # See model parameters
     pprint(lda_model.get_params())
 
+    # switching to the best model
     best_lda_model = lda_model
 
+    print('[x] Getting LDA output')
     lda_output = best_lda_model.transform(data_vectorized)
 
     topicnames = ["Topic" + str(i) for i in range(best_lda_model.n_components)]
@@ -96,8 +100,6 @@ def main(choose_context):
     # Assign Column and Index
     df_topic_keywords.columns = vectorizer.get_feature_names_out()
     df_topic_keywords.index = topicnames
-    # View
-    df_topic_keywords
 
     # Show top n keywords for each topic
     def show_topics(vectorizer=vectorizer, lda_model=lda_model, n_words=20):
@@ -190,25 +192,28 @@ def main(choose_context):
     return fig
 
 
+# def main():
+
+    
+
+
 with gr.Blocks() as demo:
-    gr.Markdown("## Dashboard per l'analisi di LDA")
-    gr.Markdown("#### Questo è un sottotitolo")
-    button = gr.Radio(
-        label="Plot type",
-        choices=['scatter_plot', 'heatmap', 'us_map', 'interactive_barplot', "radial", "multiline"], value='scatter_plot'
-    )
+    gr.Markdown("# Dashboard per l'analisi con LDA")
+    gr.Markdown("### Questo è un sottotitolo")
     # gradio.Dataframe(···)
-    choose_context = gr.Radio(
-        label="Context LDA",
-        choices=['comment', 'sup comment', 'sup comment + comment'], value='sup comment'
-    )
-    plot = gr.Plot(label="Plot")
-    choose_context.change(main, inputs=[choose_context], outputs=[plot])
+    
+    n_comp = gr.Slider(2, 25, value=5, step = 1, label="N components", info="Scegli il numero di componenti per LDA"),
 
     btn = gr.Button(value="Submit")
-    btn.click(main, inputs=[choose_context], outputs=[plot])
+    
+    plot = gr.Plot(label="Plot")
+
+    btn.click(get_lda, inputs=[n_comp[0]], outputs=[plot])
+
+    
+    
 
-    demo.load(main, inputs=[choose_context], outputs=[plot])
+    # demo.load(main, inputs=[], outputs=[plot])
 
 
 # iface = gr.Interface(fn=greet, inputs="text", outputs="text")