Update app.py

app.py

@@ -2,15 +2,15 @@ import streamlit as st
 import pandas as pd
 import xml.etree.ElementTree as ET
 import requests
-from datetime import datetime
-import pytz
+from datetime import datetime, timedelta
+from streamlit_autorefresh import st_autorefresh
+import json
 import hmac
 import os
 import time
 from PIL import Image
-from trend_crawl import get_trends
+from azure.cosmos import CosmosClient
 
-import re
 
 TREND_TOPICS = {
     1: "Autos and Vehicles",
@@ -34,12 +34,29 @@ TREND_TOPICS = {
     19: "Travel and Transportation"
 }
 
+
+client = CosmosClient(os.getenv("COSMOS_ENDPOINT"), os.getenv("COSMOS_KEY"))
+database = client.get_database_client(os.getenv("DATABASE_NAME"))
+container = database.get_container_client(os.getenv("CONTAINER_NAME"))
+
+
+
 def parse_url(url):
     response = requests.get(url)
 
     root = ET.fromstring(response.content)
     return root
 
+
+def get_latest_trend(country: str):
+    """
+    Gibt das neueste Dokument für das angegebene Land zurück,
+    basierend auf dem timestamp-Feld.
+    """
+    query = f"SELECT * FROM c WHERE c.country = '{country}' ORDER BY c.timestamp DESC"
+    items = list(container.query_items(query=query, enable_cross_partition_query=True))
+    return items[0] if items else None
+
 def convert_into_dict(req_json):
     result = {}
 
@@ -117,69 +134,15 @@ def find_details2(req_json):
 if 'reset' not in st.session_state:
     st.session_state.reset = False
 
-def display_trends_from_yesterday():
-    checkbox_statuses = {}
-    urls = []
-    
-    timezone = 'Europe/Vienna'
-    today = datetime.now(pytz.timezone(timezone)).date()
-    feed = parse_url(feed_url1)
-    entries = []
-    ns = {'ht': 'https://trends.google.de/trends/trendingsearches/daily'}  # Define namespace
-    for item in feed.findall('.//item'):
-        pubDate = datetime.strptime(item.find('pubDate').text, '%a, %d %b %Y %H:%M:%S %z').date()
-        # Filter: Überspringe, wenn pubDate heute ist
-        if pubDate == today:
-            continue
-        entry = {
-            'title': item.find('title').text,
-            'pubDate': item.find('pubDate').text,
-            'approx_traffic': item.find('ht:approx_traffic', ns).text if item.find('ht:approx_traffic', ns) is not None else None,
-            'news_items': []
-        }
-        for news_item in item.findall('ht:news_item', ns):
-            news_details = {
-                'title': news_item.find('ht:news_item_title', ns).text,
-                'snippet': news_item.find('ht:news_item_snippet', ns).text,
-                'url': news_item.find('ht:news_item_url', ns).text,
-                'source': news_item.find('ht:news_item_source', ns).text
-            }
-            entry['news_items'].append(news_details)
-        entries.append(entry)
-    
-    count = 1
-    for entry in entries:
-        with st.expander(f"{count}• {entry['title']} | Generated Traffic: {entry['approx_traffic']}"):
-            st.write(f"Veröffentlichungsdatum : {entry['pubDate']}")
-            for count2, link in enumerate(entry['news_items'], start=1):
-                checkbox_label = f"yesterday_{count}_{count2}"
-                if st.session_state.reset:
-                    st.session_state[checkbox_label] = False
-                checkbox_statuses[checkbox_label] = st.session_state.get(checkbox_label, False)
-                checkbox_statuses[checkbox_label] = st.checkbox(
-                    f"{count2}•    {link['title']} | {link['source']} | [Go To →]({link['url']})",
-                    value=checkbox_statuses[checkbox_label],
-                    key=checkbox_label
-                )
-                if checkbox_statuses[checkbox_label]:
-                    urls.append(link['url'])
-
-            # Button am Ende des Expanders
-            base_url = os.getenv("url")
-            query_params = "&".join([f"article-links[]={url}" for url in urls])
-            full_url = f"{base_url}{query_params}"
-            st.link_button("Open All Links" , url= full_url)
-            count += 1
-            
-
 
 # Function to display articles for a specific category
 def display_articles_for_category(category):
     checkbox_statuses = {}
     urls = []
 
-    trending_data = st.session_state["real_trending_searches"][selected_country][selected_country][category]
+    trending_data = st.session_state["real_trending_searches"][selected_country][category]
     #st.write(trending_data)
+    active_string = " | :chart_with_upwards_trend: :green[Aktiver Trend]"
     if st.session_state.get("reset", False):
         for idx, (topic, data) in enumerate(trending_data.items()):
             for article_index, _ in enumerate(data["articles"]):
@@ -188,8 +151,12 @@ def display_articles_for_category(category):
 
     for idx, (topic, data) in enumerate(trending_data.items()):
 
+        with st.expander(f"{idx + 1}•   {topic}" + (active_string if not data["is_trend_finished"] else "")):
 
-        with st.expander(f"{idx + 1}•   {topic} | Generated Traffic: {data['searchQueries']}"):
+            if data["related_queries"]:
+                st.markdown("**Related Keywords:**")
+                related_queries_string = f"{', '.join(data['related_queries'])}"
+                st.markdown(related_queries_string)
 
             for article_index, article in enumerate(data["articles"], start=1):
                 checkbox_label = f"{category}_{idx}_{article_index}"
@@ -201,8 +168,12 @@ def display_articles_for_category(category):
 
                 disabled = (not current_value) and (sum(checkbox_statuses.values()) >= MAX_CHECKED)
 
+                dt = datetime.fromisoformat(article["time"])
+                formatted_dt = dt.strftime("%d.%m.%Y %H:%M")
+                st.markdown(f"{article_index}•    {article['source']} / {formatted_dt}:")
+
                 checkbox_statuses[checkbox_label] = st.checkbox(
-                    f"{article_index}•    {article['title']} | [Go To →]({article['href']})",
+                    f"{article['title']} | [Go To →]({article['href']})",
                     value=current_value,
                     key=checkbox_label,
                     disabled=disabled
@@ -216,39 +187,6 @@ def display_articles_for_category(category):
             full_url = f"{base_url}{query_params}"
             st.link_button("Open All Links", url=full_url)
 
-# Funktion zum Rendern von Artikeln für heute
-def display_articles_for_today(count, index):
-    checkbox_statuses = {}
-    urls = []
-    # Dictionary zur Verwaltung des Status jeder Checkbox
-    for count2, url in enumerate(index['articles'], start=1):
-        checkbox_label = f"today_{count}_{count2}"
-        if st.session_state.reset:
-            st.session_state[checkbox_label] = False
-        checkbox_statuses[checkbox_label] = st.session_state.get(checkbox_label, False)
-
-
-    with st.expander(f"{count+1}•    {index['title']['query']} | Generated Traffic: {index['formattedTraffic']}"):
-        articles = find_details2(index['articles'])
-        for count2, url in enumerate(articles, start=1):
-            checkbox_label = f"today_{count}_{count2}"
-            disabled = not checkbox_statuses[checkbox_label] and sum(checkbox_statuses.values()) >= MAX_CHECKED
-            checkbox_statuses[checkbox_label] = st.checkbox(
-                f"{count2}•    {url['articleTitle']} | {url['source']} | [Go To →]({url['url']})",
-                value=checkbox_statuses[checkbox_label],
-                key=checkbox_label,
-                disabled=disabled
-            )
-            if checkbox_statuses[checkbox_label]:
-                urls.append(url['url'])
-
-        # Button am Ende des Expanders
-        base_url = os.getenv("url")
-        query_params = "&".join([f"article-links[]={url}" for url in urls])
-        full_url = f"{base_url}{query_params}"
-        st.link_button("Open All Links" , url= full_url)
-        
-
 country_list = {
     "Germamy" : "DE",
     "Austria" : "AT"
@@ -259,16 +197,24 @@ if 'base_load_finished' not in st.session_state:
     st.session_state["base_data"] = {}
     st.session_state["pn"] = "AT"
 if 'base_load_finished' not in st.session_state or st.session_state.reset:
+
     with st.spinner("Loading Trends"):
         st.session_state["today"] = {}
         st.session_state["base"] = {}
         for country_name, pn_option in country_list.items():
             st.session_state["base_data"][pn_option] = {}
             st.session_state["real_trending_searches"][pn_option] = {}
-            print("Load Start")
-            st.session_state["real_trending_searches"][pn_option] = get_trends(pn_option) 
-
+            with open(f"data_{pn_option}.json", "r") as file:
+                data = json.load(file)
+            try:
+                st.session_state["real_trending_searches"][pn_option] = get_latest_trend(pn_option).get("trends")
+            except:
+                st.warning("No data found in the Trendcrawl"
+                           )
+                st.stop()
+        st.session_state["_"] = 0
         st.session_state["base_load_finished"]= True
+        st.session_state["start_time"] = datetime.now()
 
 
 MAX_CHECKED = 3
@@ -328,42 +274,60 @@ fixed_order = [
 if 'selected_option' not in st.session_state:
     st.session_state['selected_option'] = "default_value"  
 
+
 img = Image.open(r"heute_tensora.png")
 st.sidebar.image(img)
+elapsed_time = datetime.now() - st.session_state["start_time"]
+if elapsed_time > timedelta(minutes=10):
+    st.markdown(
+        """
+        <style>
+        /* Zentriert den Button und vergrößert ihn */
+        div.stButton > button {
+            width: 300px;
+            height: 70px;
+            font-size: 20px;
+            margin: auto;
+            display: block;
+        }
+        </style>
+        """,
+        unsafe_allow_html=True
+    )
+
+    if st.button(f"Letzte Aktualisierung {st.session_state['start_time'].strftime('%H:%M')}\n-  Für Update Hier Klicken"):
+        st.session_state["reset"] = True
+        st.session_state["base_load_finished"] = False
+        st.rerun()
+
 
 # Selectbox to choose a country
 selected_country = st.sidebar.selectbox("Choose a Country", ["AT", "DE"])
-feed_url1 = f'https://trends.google.de/trends/trendingsearches/daily/rss?geo={selected_country}'
 
-# Button to trigger actions
-if st.sidebar.button("Change Country"):
-    if selected_country == "AT":
-        st.session_state["pn"] = selected_country
-    elif selected_country == "DE":
-        st.session_state["pn"] = selected_country
 
-selected_option = st.sidebar.radio("Choose an option", ["Realzeit Anfragen", "Trends von Gestern"])#, "Tagesaktuelle Anfragen"
 st.warning("Die aufgelisteten Keywörter für erhöhte Reichweite in den Überschriften verwenden")
 
-#if selected_option == "Tagesaktuelle Anfragen":    
-#
-#    for count, index in enumerate(st.session_state["today"][selected_country], start=0):
-#        try:
-#            display_articles_for_today(count, index)
-#        except Exception as e:
-#            st.code(e)
-#            continue
-if selected_option == "Realzeit Anfragen":
-    available_topics = st.session_state["real_trending_searches"][selected_country][selected_country].keys()
-    filtered_topics = [topic for topic in TREND_TOPICS.values() if topic in available_topics]
-    all_topics = ["All categories"] + filtered_topics
-    auswahl = st.selectbox("Select Ressort", all_topics, index=0)
-
-    display_articles_for_category(auswahl)
-
-elif selected_option == "Trends von Gestern":
-    st.error("Aktuell nicht Erreichbar")
-    #display_trends_from_yesterday()
+st.warning("Der Hinweis „Aktiver Trend“ signalisiert, dass zurzeit signifikant mehr Suchanfragen zu diesem Thema erfolgen. Bitte beachten Sie, dass auch andere Themen weiterhin von Relevanz sind.")
+
+available_topics = st.session_state["real_trending_searches"][selected_country].keys()
+filtered_topics = [topic for topic in TREND_TOPICS.values() if topic in available_topics]
+all_topics = ["All categories"] + filtered_topics
+auswahl = st.selectbox("Select Ressort", all_topics, index=0)
+
+display_articles_for_category(auswahl)
+
+if st.session_state["base_load_finished"]:
+    st.session_state["_"] = st_autorefresh(interval=600000, limit=5, key=f"autorefresh_{st.session_state['start_time']}")
+    st.markdown(
+    """
+    <style>
+    [data-testid="stToolbar"] {
+        display: none;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True  
+    )
 
 if st.session_state.reset:
     st.session_state["reset"] = False