updated news parsing

app.py

@@ -46,37 +46,33 @@ with st.sidebar:
 
 
 if st.button("Search"):
+  classifier = pipeline(task="text-classification", model=settings["model"], top_k=None)
   # display a loading progress
   with st.spinner("Loading last news ..."):
-    classifier = pipeline(task="text-classification", model=settings["model"], top_k=None)
-    df = wna.get_news(settings, query)
+    allnews = wna.get_news(settings, query)
+    st.dataframe(allnews)
   with st.spinner("Processing received news ..."):
-    # st.dataframe(df)
-    # get each title colums
-    sentences = df["title"]
-    # convert into array
-    sentences = sentences.tolist()
-    # st.write(sentences)
-    # create new dataframe
-    df = pd.DataFrame(columns=["sentence", "best","second"])
+    df = pd.DataFrame(columns=["sentence", "date","best","second"])
     # loop on each sentence and call classifier
-    for sentence in sentences:
-      cur_sentence = sentence
-      model_outputs = classifier(sentence)
+    for curnews in allnews:
+      #st.write(curnews)
+      cur_sentence = curnews["title"]
+      cur_date = curnews["date"]
+      model_outputs = classifier(cur_sentence)
       cur_result = model_outputs[0]
       #st.write(cur_result)
       # get label 1
       label = cur_result[0]['label']
       score = cur_result[0]['score']
       percentage = round(score * 100, 2)
-      str1 = label + " " + str(percentage)
+      str1 = label + " (" + str(percentage) + ")%"
       # get label 2
       label = cur_result[1]['label']
       score = cur_result[1]['score']
       percentage = round(score * 100, 2)
-      str2 = label + " " + str(percentage)
+      str2 = label + " (" + str(percentage) + ")%"
       # insert cur_sentence and cur_result into dataframe
-      df.loc[len(df.index)] = [cur_sentence, str1, str2]
+      df.loc[len(df.index)] = [cur_sentence, cur_date, str1, str2]
 
   # write info on the output
   st.write("Number of sentences:", len(df))

wna_googlenews.py

@@ -19,5 +19,4 @@ def get_news(settings, query):
       page_result = googlenews.page_at(page)
       # merge dat
       final_list = final_list + page_result
-  df = pd.DataFrame(final_list)
-  return df
+  return final_list