Fix: JSON syntax error and deploy all changes maj async auto serch

app.py

@@ -307,34 +307,46 @@ async def gradio_search(query: str, pays: str, langue: str, fuzzy: bool):
         titre = doc.get('titre', 'Sans titre')
         texte = doc.get('texte', '')[:200] + "..."
         pays_doc = doc.get('pays', 'Inconnu')
+        source = doc.get('source_url', '#')
         score = doc.get('_score', 0)
+        date = doc.get('date', '').split('T')[0]
         
         output += f"### {i}. {titre}\n"
-        output += f"**Pays:** {pays_doc}\n\n"
+        output += f"**Pays:** {pays_doc} | **Date:** {date}\n\n"
         output += f"{texte}\n\n"
-        output += f"[🔗 Source]({source})\n\n"
+        output += f"[🔗 Lire la source]({source})\n\n"
         output += "---\n\n"
     
     return output
 
-def gradio_stats():
+async def gradio_stats():
     """Affiche les statistiques pour Gradio"""
-    stats = search_engine.get_stats()
+    # Fix: await pour la fonction async
+    stats = await search_engine.get_stats()
     
     output = "# 📊 Statistiques de la Base de Données\n\n"
     output += f"**Total de documents:** {stats['total_documents']}\n\n"
     
     output += "## 🌍 Répartition par Pays\n\n"
-    for pays, count in sorted(stats['pays'].items(), key=lambda x: x[1], reverse=True):
-        output += f"- **{pays}:** {count} documents\n"
+    if stats['pays']:
+        for pays, count in sorted(stats['pays'].items(), key=lambda x: x[1], reverse=True):
+            output += f"- **{pays}:** {count} documents\n"
+    else:
+        output += "_Aucune donnée_\n"
     
     output += "\n## 🗣️ Répartition par Langue\n\n"
-    for langue, count in sorted(stats['langues'].items(), key=lambda x: x[1], reverse=True):
-        output += f"- **{langue}:** {count} documents\n"
+    if stats['langues']:
+        for langue, count in sorted(stats['langues'].items(), key=lambda x: x[1], reverse=True):
+            output += f"- **{langue}:** {count} documents\n"
+    else:
+        output += "_Aucune donnée_\n"
     
     output += "\n## 📰 Répartition par Source\n\n"
-    for source, count in sorted(stats['sources'].items(), key=lambda x: x[1], reverse=True)[:10]:
-        output += f"- **{source}:** {count} documents\n"
+    if stats['sources']:
+        for source, count in sorted(stats['sources'].items(), key=lambda x: x[1], reverse=True)[:10]:
+            output += f"- **{source}:** {count} documents\n"
+    else:
+        output += "_Aucune donnée_\n"
     
     return output
 

scraper/main.py

@@ -11,7 +11,8 @@ from utils.config import STORAGE_PATH, SCRAPER_DELAY, SCRAPER_CONCURRENT_REQUEST
 from utils.logger import setup_logger
 from db.postgres_connector import SessionLocal
 from db.models import Document, DocumentVersion
-from db.mongo_connector import save_to_mongo
+from db.mongo_connector import save_to_mongo, db
+
 from parser.cleaner import clean_html
 from indexer.typesense_indexer import index_document as index_typesense
 from utils.uuid_gen import generate_uuid
@@ -25,7 +26,7 @@ class ScrapDjiScraper:
         self.sem = asyncio.Semaphore(10)  # Augmenté pour scraping massif
         self.buffer = []
         self.buffer_size = 50  # Buffer augmenté pour scraping massif
-        self.discovered_urls = set()  # Pour éviter les doublons
+        self.discovered_urls = set()  # Pour éviter les doublons (mémoire session)
 
     def load_sources(self) -> Dict:
         if not os.path.exists(self.sources_file):
@@ -33,6 +34,16 @@ class ScrapDjiScraper:
         with open(self.sources_file, 'r', encoding='utf-8') as f:
             return json.load(f)
 
+    async def is_url_scraped(self, url: str) -> bool:
+        """Vérifie si l'URL existe déjà dans MongoDB"""
+        try:
+            # Vérification rapide sur l'ID (source_url est unique ?)
+            # Ou count_documents
+            doc = await db["documents"].find_one({"source_url": url}, {"_id": 1})
+            return doc is not None
+        except Exception:
+            return False
+
     async def scrape_article(self, client: httpx.AsyncClient, source: Dict, url: str) -> Optional[Dict]:
         """Scrape ultra-rapide avec lxml (C-level parsing)"""
         try:
@@ -77,7 +88,7 @@ class ScrapDjiScraper:
             loop = asyncio.get_event_loop()
             links = await loop.run_in_executor(None, self._extract_links_sync, resp.content, base_url)
             
-            # Filtrage rapide (peut rester dans le thread principal ou migrer si très lourd)
+            # Filtrage rapide
             new_links = []
             for link in links:
                 if link not in self.discovered_urls:
@@ -162,7 +173,8 @@ class ScrapDjiScraper:
         count = 0
         async with self.sem:
             try:
-                # 1. Scraper la page principale
+                # 1. Scraper la page principale (si non scrapée récemment?)
+                # On ne vérifie pas is_url_scraped pour la home car elle change
                 doc = await self.scrape_article(client, source, source['url'])
                 if doc:
                     self.buffer.append(doc)
@@ -175,6 +187,11 @@ class ScrapDjiScraper:
                 
                 # 3. Scraper les articles découverts (avec limite)
                 for url in article_urls[:50]:  # Limiter à 50 articles par source pour commencer
+                    # Vérification anti-doublon MongoDB
+                    if await self.is_url_scraped(url):
+                        # logger.debug(f"⏭️ Déjà scrapé: {url}")
+                        continue
+                    
                     doc = await self.scrape_article(client, source, url)
                     if doc:
                         self.buffer.append(doc)

sources.json

@@ -16,6 +16,14 @@
       "langue": "fr",
       "active": true
     },
+    {
+      "name": "27avril",
+      "type": "news",
+      "url": "https://www.27avril.com",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
     {
       "name": "IciLome",
       "type": "news",
@@ -40,6 +48,14 @@
       "langue": "fr",
       "active": true
     },
+    {
+      "name": "RepubliqueTogolaise",
+      "type": "news",
+      "url": "https://www.republiquetogolaise.com",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
     {
       "name": "TogoActualite",
       "type": "news",
@@ -64,6 +80,46 @@
       "langue": "fr",
       "active": true
     },
+    {
+      "name": "ActuLome",
+      "type": "news",
+      "url": "https://actulome.com",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
+    {
+      "name": "24HeureInfo",
+      "type": "news",
+      "url": "https://24heureinfo.com",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
+    {
+      "name": "RadioLome",
+      "type": "news",
+      "url": "https://www.radiolome.tg",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
+    {
+      "name": "aLome",
+      "type": "news",
+      "url": "http://www.alome.com",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
+    {
+      "name": "TogoTopNews",
+      "type": "news",
+      "url": "https://togotopnews.tg",
+      "pays": "Togo",
+      "langue": "fr",
+      "active": true
+    },
     {
       "name": "BeninWebTV",
       "type": "news",