Spaces:
Sleeping
Sleeping
Update src/streamlit_app.py
Browse files- src/streamlit_app.py +24 -57
src/streamlit_app.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# Fichier: app.py
|
| 2 |
|
| 3 |
# ===================================================================================
|
| 4 |
# WAHIS SCRAPER - VERSION TABLEAU DE BORD STREAMLIT (APPROCHE FINALE ET ROBUSTE)
|
|
@@ -11,17 +11,16 @@ from datetime import datetime
|
|
| 11 |
from pathlib import Path
|
| 12 |
import warnings
|
| 13 |
import asyncio
|
| 14 |
-
import subprocess
|
| 15 |
from playwright.async_api import async_playwright
|
| 16 |
from playwright_stealth import stealth_async
|
| 17 |
from streamlit_folium import st_folium
|
| 18 |
import folium
|
|
|
|
| 19 |
|
| 20 |
# --- Configuration de la Page Streamlit ---
|
| 21 |
st.set_page_config(layout="wide", page_title="WAHIS Animal Disease Dashboard")
|
| 22 |
|
| 23 |
# --- Classes et Fonctions de Scraping (inchangées) ---
|
| 24 |
-
# ... (la logique de scraping est parfaite, on la garde telle quelle)
|
| 25 |
class WAHISScraper:
|
| 26 |
def __init__(self): self.logs = []
|
| 27 |
def log(self, message):
|
|
@@ -34,6 +33,7 @@ class WAHISScraper:
|
|
| 34 |
browser = None
|
| 35 |
try:
|
| 36 |
self.log("🔧 Lancement d'un navigateur Chromium...")
|
|
|
|
| 37 |
browser = await p.chromium.launch(headless=True, args=["--no-sandbox"])
|
| 38 |
page = await browser.new_page()
|
| 39 |
self.log("🕵️ Application du camouflage 'stealth'...")
|
|
@@ -65,28 +65,22 @@ class WAHISScraper:
|
|
| 65 |
finally:
|
| 66 |
if browser and browser.is_connected(): await browser.close()
|
| 67 |
|
| 68 |
-
# --- Fonctions de Traitement des Données ---
|
| 69 |
def process_data(reports, outbreaks, additional_infos):
|
| 70 |
valid_additional_infos = [info for info in additional_infos if isinstance(info, dict)]
|
| 71 |
additional_info_map = {info.get('outbreakId'): info for info in valid_additional_infos}
|
| 72 |
-
|
| 73 |
report_map = {report['eventId']: {'disease': report['disease']} for report in reports}
|
| 74 |
-
|
| 75 |
for outbreak in outbreaks:
|
| 76 |
event_info = report_map.get(outbreak.get('eventId'), {})
|
| 77 |
outbreak['diseaseName'] = event_info.get('disease')
|
| 78 |
outbreak_id = outbreak.get('outbreakId')
|
| 79 |
-
if outbreak_id in additional_info_map:
|
| 80 |
-
outbreak.update(additional_info_map[outbreak_id])
|
| 81 |
-
|
| 82 |
return pd.DataFrame(outbreaks)
|
| 83 |
|
| 84 |
# --- Construction de l'Interface Streamlit ---
|
| 85 |
|
|
|
|
| 86 |
st.title("🤖 Tableau de Bord WAHIS")
|
| 87 |
-
st.info("Ce tableau de bord extrait et affiche les derniers foyers de maladies animales signalés à l'Organisation Mondiale de la Santé Animale (WOAH).")
|
| 88 |
|
| 89 |
-
# Utilisation du "session_state" pour stocker les données après le premier chargement
|
| 90 |
if 'df_outbreaks' not in st.session_state:
|
| 91 |
st.session_state.df_outbreaks = pd.DataFrame()
|
| 92 |
st.session_state.logs = ""
|
|
@@ -95,71 +89,44 @@ if 'df_outbreaks' not in st.session_state:
|
|
| 95 |
if st.button("🚀 Lancer l'extraction des données"):
|
| 96 |
with st.spinner("Extraction en cours... (cela peut prendre 2-3 minutes)"):
|
| 97 |
scraper = WAHISScraper()
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
if not st.session_state.df_outbreaks.empty:
|
| 108 |
df = st.session_state.df_outbreaks
|
| 109 |
-
|
| 110 |
-
# --- Barre Latérale avec les Filtres ---
|
| 111 |
st.sidebar.header("🔍 Filtres")
|
| 112 |
all_diseases = ["Toutes"] + sorted(df['diseaseName'].dropna().unique())
|
| 113 |
all_species = ["Toutes"] + sorted(df['species'].dropna().unique())
|
| 114 |
-
|
| 115 |
selected_disease = st.sidebar.selectbox("Filtrer par Maladie", all_diseases)
|
| 116 |
selected_species = st.sidebar.selectbox("Filtrer par Espèce", all_species)
|
| 117 |
|
| 118 |
-
# Filtrage du DataFrame
|
| 119 |
filtered_df = df.copy()
|
| 120 |
-
if selected_disease != "Toutes":
|
| 121 |
-
|
| 122 |
-
if selected_species != "Toutes":
|
| 123 |
-
filtered_df = filtered_df[filtered_df['species'] == selected_species]
|
| 124 |
|
| 125 |
-
# --- Affichage Principal : Carte et Détails ---
|
| 126 |
st.header(f"🗺️ Carte de {len(filtered_df)} Foyers")
|
| 127 |
-
|
| 128 |
if filtered_df.empty or not all(k in filtered_df for k in ['latitude', 'longitude']):
|
| 129 |
-
st.warning("Aucun foyer ne correspond à vos filtres
|
| 130 |
else:
|
| 131 |
-
# Création de la carte avec Folium
|
| 132 |
m = folium.Map(location=[filtered_df['latitude'].mean(), filtered_df['longitude'].mean()], zoom_start=2)
|
| 133 |
-
|
| 134 |
for _, row in filtered_df.iterrows():
|
| 135 |
-
popup_html = f""
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
<b>Espèce:</b> {row.get('species', 'N/A')}<br>
|
| 139 |
-
<b>Cas:</b> {row.get('cases', 0)} | <b>Morts:</b> {row.get('deaths', 0)}
|
| 140 |
-
"""
|
| 141 |
-
iframe = folium.IFrame(popup_html, width=250, height=100)
|
| 142 |
-
popup = folium.Popup(iframe, max_width=250)
|
| 143 |
-
folium.Marker(
|
| 144 |
-
location=[row['latitude'], row['longitude']],
|
| 145 |
-
popup=popup,
|
| 146 |
-
tooltip=row.get('diseaseName', 'N/A')
|
| 147 |
-
).add_to(m)
|
| 148 |
-
|
| 149 |
-
# Affichage de la carte et récupération du dernier point cliqué
|
| 150 |
-
map_data = st_folium(m, width='100%')
|
| 151 |
-
if map_data and map_data['last_object_clicked_popup']:
|
| 152 |
-
st.session_state.last_click = map_data['last_object_clicked_popup']['html']
|
| 153 |
|
| 154 |
-
# Affichage des détails du dernier point cliqué
|
| 155 |
-
if st.session_state.last_click:
|
| 156 |
-
st.header("📋 Détails du Foyer Sélectionné")
|
| 157 |
-
st.markdown(st.session_state.last_click, unsafe_allow_html=True)
|
| 158 |
-
|
| 159 |
-
# --- Affichage du tableau de données ---
|
| 160 |
with st.expander("Voir le tableau de données des foyers filtrés"):
|
| 161 |
st.dataframe(filtered_df)
|
| 162 |
|
| 163 |
-
# --- Affichage des logs ---
|
| 164 |
with st.expander("Voir le journal d'exécution"):
|
| 165 |
st.text_area("Logs", st.session_state.logs, height=300)
|
|
|
|
| 1 |
+
# Fichier: src/app.py
|
| 2 |
|
| 3 |
# ===================================================================================
|
| 4 |
# WAHIS SCRAPER - VERSION TABLEAU DE BORD STREAMLIT (APPROCHE FINALE ET ROBUSTE)
|
|
|
|
| 11 |
from pathlib import Path
|
| 12 |
import warnings
|
| 13 |
import asyncio
|
|
|
|
| 14 |
from playwright.async_api import async_playwright
|
| 15 |
from playwright_stealth import stealth_async
|
| 16 |
from streamlit_folium import st_folium
|
| 17 |
import folium
|
| 18 |
+
import traceback
|
| 19 |
|
| 20 |
# --- Configuration de la Page Streamlit ---
|
| 21 |
st.set_page_config(layout="wide", page_title="WAHIS Animal Disease Dashboard")
|
| 22 |
|
| 23 |
# --- Classes et Fonctions de Scraping (inchangées) ---
|
|
|
|
| 24 |
class WAHISScraper:
|
| 25 |
def __init__(self): self.logs = []
|
| 26 |
def log(self, message):
|
|
|
|
| 33 |
browser = None
|
| 34 |
try:
|
| 35 |
self.log("🔧 Lancement d'un navigateur Chromium...")
|
| 36 |
+
# L'argument --no-sandbox est crucial dans les environnements conteneurisés
|
| 37 |
browser = await p.chromium.launch(headless=True, args=["--no-sandbox"])
|
| 38 |
page = await browser.new_page()
|
| 39 |
self.log("🕵️ Application du camouflage 'stealth'...")
|
|
|
|
| 65 |
finally:
|
| 66 |
if browser and browser.is_connected(): await browser.close()
|
| 67 |
|
|
|
|
| 68 |
def process_data(reports, outbreaks, additional_infos):
|
| 69 |
valid_additional_infos = [info for info in additional_infos if isinstance(info, dict)]
|
| 70 |
additional_info_map = {info.get('outbreakId'): info for info in valid_additional_infos}
|
|
|
|
| 71 |
report_map = {report['eventId']: {'disease': report['disease']} for report in reports}
|
|
|
|
| 72 |
for outbreak in outbreaks:
|
| 73 |
event_info = report_map.get(outbreak.get('eventId'), {})
|
| 74 |
outbreak['diseaseName'] = event_info.get('disease')
|
| 75 |
outbreak_id = outbreak.get('outbreakId')
|
| 76 |
+
if outbreak_id in additional_info_map: outbreak.update(additional_info_map[outbreak_id])
|
|
|
|
|
|
|
| 77 |
return pd.DataFrame(outbreaks)
|
| 78 |
|
| 79 |
# --- Construction de l'Interface Streamlit ---
|
| 80 |
|
| 81 |
+
st.set_page_config(layout="wide", page_title="WAHIS Animal Disease Dashboard")
|
| 82 |
st.title("🤖 Tableau de Bord WAHIS")
|
|
|
|
| 83 |
|
|
|
|
| 84 |
if 'df_outbreaks' not in st.session_state:
|
| 85 |
st.session_state.df_outbreaks = pd.DataFrame()
|
| 86 |
st.session_state.logs = ""
|
|
|
|
| 89 |
if st.button("🚀 Lancer l'extraction des données"):
|
| 90 |
with st.spinner("Extraction en cours... (cela peut prendre 2-3 minutes)"):
|
| 91 |
scraper = WAHISScraper()
|
| 92 |
+
try:
|
| 93 |
+
reports, outbreaks, additional, logs = asyncio.run(scraper.run_extraction_async())
|
| 94 |
+
if reports:
|
| 95 |
+
st.session_state.df_outbreaks = process_data(reports, outbreaks, additional)
|
| 96 |
+
st.session_state.logs = logs
|
| 97 |
+
st.success("Extraction terminée avec succès !")
|
| 98 |
+
st.experimental_rerun() # Rafraîchir l'interface pour afficher les résultats
|
| 99 |
+
else:
|
| 100 |
+
st.error("L'extraction a échoué. Veuillez consulter les logs.")
|
| 101 |
+
st.session_state.logs = logs
|
| 102 |
+
except Exception as e:
|
| 103 |
+
st.error("Une erreur critique est survenue pendant l'exécution.")
|
| 104 |
+
st.text(traceback.format_exc())
|
| 105 |
|
| 106 |
if not st.session_state.df_outbreaks.empty:
|
| 107 |
df = st.session_state.df_outbreaks
|
|
|
|
|
|
|
| 108 |
st.sidebar.header("🔍 Filtres")
|
| 109 |
all_diseases = ["Toutes"] + sorted(df['diseaseName'].dropna().unique())
|
| 110 |
all_species = ["Toutes"] + sorted(df['species'].dropna().unique())
|
|
|
|
| 111 |
selected_disease = st.sidebar.selectbox("Filtrer par Maladie", all_diseases)
|
| 112 |
selected_species = st.sidebar.selectbox("Filtrer par Espèce", all_species)
|
| 113 |
|
|
|
|
| 114 |
filtered_df = df.copy()
|
| 115 |
+
if selected_disease != "Toutes": filtered_df = filtered_df[filtered_df['diseaseName'] == selected_disease]
|
| 116 |
+
if selected_species != "Toutes": filtered_df = filtered_df[filtered_df['species'] == selected_species]
|
|
|
|
|
|
|
| 117 |
|
|
|
|
| 118 |
st.header(f"🗺️ Carte de {len(filtered_df)} Foyers")
|
|
|
|
| 119 |
if filtered_df.empty or not all(k in filtered_df for k in ['latitude', 'longitude']):
|
| 120 |
+
st.warning("Aucun foyer ne correspond à vos filtres.")
|
| 121 |
else:
|
|
|
|
| 122 |
m = folium.Map(location=[filtered_df['latitude'].mean(), filtered_df['longitude'].mean()], zoom_start=2)
|
|
|
|
| 123 |
for _, row in filtered_df.iterrows():
|
| 124 |
+
popup_html = f"<b>Lieu:</b> {row.get('locationName', 'N/A')}<br><b>Maladie:</b> {row.get('diseaseName', 'N/A')}<br><b>Espèce:</b> {row.get('species', 'N/A')}"
|
| 125 |
+
folium.Marker(location=[row['latitude'], row['longitude']], popup=popup_html, tooltip=row.get('diseaseName', 'N/A')).add_to(m)
|
| 126 |
+
st_folium(m, width='100%')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
with st.expander("Voir le tableau de données des foyers filtrés"):
|
| 129 |
st.dataframe(filtered_df)
|
| 130 |
|
|
|
|
| 131 |
with st.expander("Voir le journal d'exécution"):
|
| 132 |
st.text_area("Logs", st.session_state.logs, height=300)
|