add: db_store.py

syscred/db_store.py

@@ -0,0 +1,354 @@
+"""
+SysCRED Storage Module - SQLite + Supabase
+==========================================
+Stocke les triplets RDF et résultats d'analyse.
+Utilise SQLite localement, avec option de sync vers Supabase.
+"""
+
+import os
+import sqlite3
+import hashlib
+import json
+from datetime import datetime
+from typing import Optional, Dict, Any, List, Tuple
+from urllib.parse import urlparse
+from pathlib import Path
+
+# Chemins
+BASE_DIR = Path(__file__).parent
+DB_PATH = BASE_DIR / "syscred_local.db"
+
+class SysCREDStore:
+    """
+    Gestionnaire de stockage pour SysCRED.
+    SQLite local avec option Supabase.
+    """
+    
+    def __init__(self, db_path: str = None, supabase_url: str = None):
+        self.db_path = db_path or str(DB_PATH)
+        self.supabase_url = supabase_url or os.getenv("DATABASE_URL")
+        self.conn = None
+        self._init_local_db()
+        
+    def _init_local_db(self):
+        """Initialise la base SQLite locale."""
+        self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        self.conn.row_factory = sqlite3.Row
+        
+        # Créer les tables
+        self.conn.executescript("""
+            -- Résultats d'analyse
+            CREATE TABLE IF NOT EXISTS analysis_results (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                url TEXT NOT NULL,
+                credibility_score REAL NOT NULL,
+                summary TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                source_reputation TEXT,
+                fact_check_count INTEGER DEFAULT 0,
+                score_details TEXT,
+                domain TEXT
+            );
+            
+            -- Triplets RDF
+            CREATE TABLE IF NOT EXISTS rdf_triples (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                subject TEXT NOT NULL,
+                predicate TEXT NOT NULL,
+                object TEXT NOT NULL,
+                object_type TEXT DEFAULT 'uri',
+                graph_name TEXT DEFAULT 'data',
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                UNIQUE(subject, predicate, object, graph_name)
+            );
+            
+            -- Sources
+            CREATE TABLE IF NOT EXISTS sources (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                domain TEXT UNIQUE NOT NULL,
+                reputation_score REAL,
+                domain_age_years REAL,
+                is_fact_checker INTEGER DEFAULT 0,
+                analysis_count INTEGER DEFAULT 0,
+                last_analyzed TIMESTAMP,
+                metadata TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            );
+            
+            -- Claims
+            CREATE TABLE IF NOT EXISTS claims (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                claim_text TEXT NOT NULL,
+                claim_hash TEXT UNIQUE,
+                source_url TEXT,
+                extracted_entities TEXT,
+                credibility_score REAL,
+                verification_status TEXT DEFAULT 'unverified',
+                evidence_count INTEGER DEFAULT 0,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            );
+            
+            -- Evidence
+            CREATE TABLE IF NOT EXISTS evidence (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                claim_id INTEGER,
+                doc_id TEXT,
+                doc_text TEXT,
+                relevance_score REAL,
+                retrieval_method TEXT DEFAULT 'bm25',
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                FOREIGN KEY (claim_id) REFERENCES claims(id)
+            );
+            
+            -- Index
+            CREATE INDEX IF NOT EXISTS idx_analysis_url ON analysis_results(url);
+            CREATE INDEX IF NOT EXISTS idx_triple_subject ON rdf_triples(subject);
+            CREATE INDEX IF NOT EXISTS idx_triple_graph ON rdf_triples(graph_name);
+            CREATE INDEX IF NOT EXISTS idx_sources_domain ON sources(domain);
+        """)
+        self.conn.commit()
+        print(f"[SysCREDStore] SQLite initialisé: {self.db_path}")
+    
+    # =========================================================================
+    # ONTOLOGY / RDF TRIPLES
+    # =========================================================================
+    
+    def sync_ontology(self, ontology_manager) -> Dict[str, int]:
+        """
+        Synchronise les graphes RDFLib vers SQLite.
+        
+        Args:
+            ontology_manager: Instance avec base_graph et data_graph
+        """
+        result = {'base_synced': 0, 'data_synced': 0}
+        
+        try:
+            # Sync base ontology
+            if hasattr(ontology_manager, 'base_graph') and ontology_manager.base_graph:
+                result['base_synced'] = self._sync_graph(
+                    ontology_manager.base_graph, 
+                    graph_name='base'
+                )
+            
+            # Sync data graph
+            if hasattr(ontology_manager, 'data_graph') and ontology_manager.data_graph:
+                result['data_synced'] = self._sync_graph(
+                    ontology_manager.data_graph,
+                    graph_name='data'
+                )
+            
+            self.conn.commit()
+            print(f"[SysCREDStore] Synced {result['base_synced']} base + {result['data_synced']} data triples")
+            
+        except Exception as e:
+            result['error'] = str(e)
+            print(f"[SysCREDStore] Sync error: {e}")
+        
+        return result
+    
+    def _sync_graph(self, graph, graph_name: str) -> int:
+        """Sync un graphe RDFLib vers SQLite."""
+        from rdflib import Literal
+        
+        count = 0
+        cursor = self.conn.cursor()
+        
+        for s, p, o in graph:
+            subject = str(s)
+            predicate = str(p)
+            obj_value = str(o)
+            obj_type = 'literal' if isinstance(o, Literal) else 'uri'
+            
+            try:
+                cursor.execute("""
+                    INSERT OR IGNORE INTO rdf_triples 
+                    (subject, predicate, object, object_type, graph_name)
+                    VALUES (?, ?, ?, ?, ?)
+                """, (subject, predicate, obj_value, obj_type, graph_name))
+                count += 1
+            except:
+                pass
+        
+        return count
+    
+    def get_triple_stats(self) -> Dict[str, int]:
+        """Statistiques des triplets."""
+        cursor = self.conn.cursor()
+        
+        cursor.execute("SELECT COUNT(*) FROM rdf_triples WHERE graph_name = 'base'")
+        base = cursor.fetchone()[0]
+        
+        cursor.execute("SELECT COUNT(*) FROM rdf_triples WHERE graph_name = 'data'")
+        data = cursor.fetchone()[0]
+        
+        return {
+            'base_triples': base,
+            'data_triples': data,
+            'total_triples': base + data
+        }
+    
+    # =========================================================================
+    # ANALYSIS RESULTS
+    # =========================================================================
+    
+    def save_analysis(self, url: str, credibility_score: float,
+                      summary: str = None, score_details: Dict = None,
+                      source_reputation: str = None, fact_check_count: int = 0) -> int:
+        """Sauvegarde un résultat d'analyse."""
+        domain = urlparse(url).netloc
+        
+        cursor = self.conn.cursor()
+        cursor.execute("""
+            INSERT INTO analysis_results 
+            (url, credibility_score, summary, score_details, source_reputation, 
+             fact_check_count, domain)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+        """, (
+            url, credibility_score, summary,
+            json.dumps(score_details) if score_details else None,
+            source_reputation, fact_check_count, domain
+        ))
+        self.conn.commit()
+        
+        result_id = cursor.lastrowid
+        print(f"[SysCREDStore] Saved analysis #{result_id} for {domain}")
+        
+        # Update source stats
+        self._update_source(domain, credibility_score)
+        
+        return result_id
+    
+    def get_history(self, url: str = None, limit: int = 50) -> List[Dict]:
+        """Récupère l'historique des analyses."""
+        cursor = self.conn.cursor()
+        
+        if url:
+            cursor.execute("""
+                SELECT * FROM analysis_results
+                WHERE url = ? ORDER BY created_at DESC LIMIT ?
+            """, (url, limit))
+        else:
+            cursor.execute("""
+                SELECT * FROM analysis_results
+                ORDER BY created_at DESC LIMIT ?
+            """, (limit,))
+        
+        return [dict(row) for row in cursor.fetchall()]
+    
+    # =========================================================================
+    # SOURCES
+    # =========================================================================
+    
+    def _update_source(self, domain: str, score: float = None):
+        """Met à jour les stats d'une source."""
+        cursor = self.conn.cursor()
+        
+        cursor.execute("SELECT id, analysis_count FROM sources WHERE domain = ?", (domain,))
+        row = cursor.fetchone()
+        
+        if row:
+            cursor.execute("""
+                UPDATE sources SET 
+                    analysis_count = analysis_count + 1,
+                    last_analyzed = CURRENT_TIMESTAMP,
+                    reputation_score = COALESCE(?, reputation_score)
+                WHERE domain = ?
+            """, (score, domain))
+        else:
+            cursor.execute("""
+                INSERT INTO sources (domain, reputation_score, analysis_count, last_analyzed)
+                VALUES (?, ?, 1, CURRENT_TIMESTAMP)
+            """, (domain, score))
+        
+        self.conn.commit()
+    
+    def get_source(self, domain: str) -> Optional[Dict]:
+        """Récupère les infos d'une source."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT * FROM sources WHERE domain = ?", (domain,))
+        row = cursor.fetchone()
+        return dict(row) if row else None
+    
+    # =========================================================================
+    # GLOBAL STATS
+    # =========================================================================
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Statistiques globales."""
+        cursor = self.conn.cursor()
+        
+        cursor.execute("SELECT COUNT(*) FROM analysis_results")
+        total_analyses = cursor.fetchone()[0]
+        
+        cursor.execute("SELECT COUNT(*) FROM sources")
+        unique_domains = cursor.fetchone()[0]
+        
+        cursor.execute("SELECT AVG(credibility_score) FROM analysis_results")
+        avg_score = cursor.fetchone()[0]
+        
+        triple_stats = self.get_triple_stats()
+        
+        return {
+            'total_analyses': total_analyses,
+            'unique_domains': unique_domains,
+            'avg_credibility': round(avg_score, 2) if avg_score else None,
+            **triple_stats
+        }
+    
+    def close(self):
+        """Ferme la connexion."""
+        if self.conn:
+            self.conn.close()
+
+
+# ============================================================================
+# INTEGRATION
+# ============================================================================
+
+def sync_ontology_to_db():
+    """Synchronise l'ontologie vers la base de données."""
+    import sys
+    sys.path.insert(0, str(BASE_DIR))
+    
+    try:
+        from ontology_manager import OntologyManager
+        from config import Config
+        
+        # Init ontology
+        onto = OntologyManager(
+            base_ontology_path=str(Config.ONTOLOGY_BASE_PATH),
+            data_path=str(Config.ONTOLOGY_DATA_PATH)
+        )
+        
+        # Init store
+        store = SysCREDStore()
+        
+        # Sync
+        result = store.sync_ontology(onto)
+        print(f"\n✅ Sync complete: {result}")
+        
+        # Stats
+        stats = store.get_stats()
+        print(f"📊 Stats: {stats}")
+        
+        return store
+        
+    except ImportError as e:
+        print(f"Import error: {e}")
+        return None
+
+
+# ============================================================================
+# CLI
+# ============================================================================
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("SysCRED Storage - Synchronisation des triplets")
+    print("=" * 60)
+    
+    store = sync_ontology_to_db()
+    
+    if store:
+        print("\n✅ Base de données prête!")
+        print(f"   Fichier: {store.db_path}")