Sync from Sandbox: fix NER, E-E-A-T, backend v2.4.1

Dockerfile

@@ -1,5 +1,5 @@
 # SysCRED Docker Configuration for Hugging Face Spaces
-# OPTIMIZED version with Distilled Models for faster startup
+# Full version with PyTorch and Transformers
 FROM python:3.10-slim
 
 WORKDIR /app
@@ -7,48 +7,27 @@ WORKDIR /app
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONPATH=/app
-
-# ============================================
-# KEY OPTIMIZATION: Use distilled models
-# ============================================
 ENV SYSCRED_LOAD_ML_MODELS=true
-ENV SYSCRED_USE_DISTILLED=true
-ENV TRANSFORMERS_CACHE=/app/.cache/huggingface
-ENV HF_HOME=/app/.cache/huggingface
+ENV SYSCRED_ENV=production
 
 # Install system dependencies
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy optimized requirements (distilled models, CPU-only torch)
-COPY syscred/requirements-distilled.txt /app/requirements.txt
+# Copy full requirements
+COPY requirements.txt /app/requirements.txt
 
-# Install dependencies
+# Install dependencies (includes PyTorch, Transformers, spaCy)
 RUN pip install --no-cache-dir -r requirements.txt
 
-# ============================================
-# PRE-DOWNLOAD DISTILLED MODELS (Build Time)
-# This avoids timeout during first request
-# ============================================
-RUN python -c "from transformers import pipeline; \
-    pipeline('sentiment-analysis', model='distilbert-base-uncased-finetuned-sst-2-english'); \
-    pipeline('ner', model='dslim/bert-base-NER'); \
-    print('✓ Distilled models pre-downloaded')"
-
-# Download small spaCy models
-RUN pip install spacy && \
-    python -m spacy download en_core_web_sm && \
-    python -m spacy download fr_core_news_sm && \
-    echo '✓ spaCy models downloaded'
-
-# Pre-download sentence transformer (small version)
-RUN python -c "from sentence_transformers import SentenceTransformer; \
-    SentenceTransformer('all-MiniLM-L6-v2'); \
-    print('✓ Sentence transformer pre-downloaded')"
+# Download spaCy models for NER
+RUN python -m spacy download en_core_web_md || true
+RUN python -m spacy download fr_core_news_md || true
 
 # Copy application code
 COPY syscred/ /app/syscred/
+COPY ontology/ /app/ontology/
 
 # Create user for HF Spaces (required)
 RUN useradd -m -u 1000 user
@@ -58,8 +37,8 @@ ENV PATH=/home/user/.local/bin:$PATH
 
 WORKDIR /app
 
+# HF Spaces uses port 7860
 EXPOSE 7860
 
-# Run with HF Spaces port (7860)
-# Increased workers to 4 for better concurrency, timeout 600s
-CMD ["gunicorn", "--bind", "0.0.0.0:7860", "--workers", "1", "--threads", "4", "--timeout", "600", "syscred.backend_app:app"]
+# Run with HF Spaces port
+CMD ["gunicorn", "--bind", "0.0.0.0:7860", "--workers", "2", "--timeout", "300", "syscred.backend_app:app"]

requirements.txt

@@ -0,0 +1,38 @@
+# SysCRED - Requirements (Full version with ML models)
+# Système Hybride de Vérification de Crédibilité
+# (c) Dominique S. Loyer
+# Version complète pour HuggingFace Spaces et développement local
+
+# === Core Dependencies ===
+requests>=2.28.0
+beautifulsoup4>=4.11.0
+python-whois>=0.8.0
+lxml>=4.9.0
+
+# === RDF/Ontology ===
+rdflib>=6.0.0
+
+# === Machine Learning ===
+transformers>=4.30.0
+torch>=2.0.0
+numpy>=1.24.0
+sentence-transformers>=2.2.0
+accelerate>=0.20.0
+spacy>=3.6.0
+
+# === Explainability ===
+lime>=0.2.0
+
+# === Web Backend ===
+flask>=2.3.0
+flask-cors>=4.0.0
+python-dotenv>=1.0.0
+pandas>=2.0.0
+
+# === Production/Database ===
+gunicorn>=20.1.0
+psycopg2-binary>=2.9.0
+flask-sqlalchemy>=3.0.0
+
+# === Development/Testing ===
+pytest>=7.0.0

syscred/backend_app.py

@@ -22,12 +22,16 @@ import traceback
 from pathlib import Path
 try:
     from dotenv import load_dotenv
-    env_path = Path(__file__).parent / '.env'
+    # .env is at project root (parent of syscred/)
+    env_path = Path(__file__).resolve().parent.parent / '.env'
+    if not env_path.exists():
+        # Fallback: check syscred/ directory
+        env_path = Path(__file__).parent / '.env'
     if env_path.exists():
         load_dotenv(env_path)
         print(f"[SysCRED Backend] Loaded .env from {env_path}")
     else:
-        print(f"[SysCRED Backend] No .env file found at {env_path}")
+        print(f"[SysCRED Backend] No .env file found, using system env vars")
 except ImportError:
     print("[SysCRED Backend] python-dotenv not installed, using system env vars")
 
@@ -85,6 +89,16 @@ except ImportError as e:
 app = Flask(__name__)
 CORS(app)  # Enable CORS for frontend
 
+# Allow iframe embedding on UQAM domains (for syscred.uqam.ca mirror)
+@app.after_request
+def add_security_headers(response):
+    """Add security headers allowing UQAM iframe embedding."""
+    response.headers['X-Frame-Options'] = 'ALLOW-FROM https://syscred.uqam.ca'
+    response.headers['Content-Security-Policy'] = (
+        "frame-ancestors 'self' https://syscred.uqam.ca https://*.uqam.ca"
+    )
+    return response
+
 # Initialize Database
 try:
     init_db(app) # [NEW] Setup DB connection
@@ -267,6 +281,62 @@ def verify_endpoint():
         
         print(f"[SysCRED Backend] Score: {result.get('scoreCredibilite', 'N/A')}")
         
+        # [NEW] TREC Evidence Search + IR Metrics
+        try:
+            global trec_retriever, eval_metrics
+            
+            # Initialize TREC if needed
+            if trec_retriever is None and TREC_AVAILABLE:
+                trec_retriever = TRECRetriever(use_stemming=True, enable_prf=False)
+                trec_retriever.corpus = TREC_DEMO_CORPUS
+                eval_metrics = EvaluationMetrics()
+                print("[SysCRED Backend] TREC Retriever initialized with demo corpus")
+            
+            if trec_retriever and eval_metrics:
+                import time
+                start_time = time.time()
+                
+                # Use the input text as query
+                query_text = input_data[:200] if not credibility_system.is_url(input_data) else result.get('informationEntree', input_data)[:200]
+                
+                trec_result = trec_retriever.retrieve_evidence(query_text, k=5, model='bm25')
+                search_time = (time.time() - start_time) * 1000
+                
+                retrieved_ids = [e.doc_id for e in trec_result.evidences]
+                
+                # Use climate-related docs as "relevant" for demo evaluation
+                # In production, this would come from qrels files
+                relevant_ids = set(TREC_DEMO_CORPUS.keys())  # All docs as relevant pool
+                
+                # Compute IR metrics
+                k = len(retrieved_ids) if retrieved_ids else 1
+                precision = eval_metrics.precision_at_k(retrieved_ids, relevant_ids, k) if retrieved_ids else 0
+                recall = eval_metrics.recall_at_k(retrieved_ids, relevant_ids, k) if retrieved_ids else 0
+                ap = eval_metrics.average_precision(retrieved_ids, relevant_ids) if retrieved_ids else 0
+                mrr = eval_metrics.mrr(retrieved_ids, relevant_ids) if retrieved_ids else 0
+                
+                relevance_dict = {doc: 1 for doc in relevant_ids}
+                ndcg = eval_metrics.ndcg_at_k(retrieved_ids, relevance_dict, k) if retrieved_ids else 0
+                
+                # TF-IDF score from top result
+                tfidf_score = trec_result.evidences[0].score if trec_result.evidences else 0
+                
+                result['trec_metrics'] = {
+                    'precision': round(precision, 4),
+                    'recall': round(recall, 4),
+                    'map': round(ap, 4),
+                    'ndcg': round(ndcg, 4),
+                    'tfidf_score': round(tfidf_score, 4),
+                    'mrr': round(mrr, 4),
+                    'retrieved_count': len(retrieved_ids),
+                    'corpus_size': len(TREC_DEMO_CORPUS),
+                    'search_time_ms': round(search_time, 2)
+                }
+                print(f"[SysCRED Backend] TREC: P={precision:.3f} R={recall:.3f} MAP={ap:.3f} NDCG={ndcg:.3f} MRR={mrr:.3f}")
+        except Exception as e:
+            print(f"[SysCRED Backend] TREC metrics error: {e}")
+            result['trec_metrics'] = {'error': str(e)}
+        
         # [NEW] Persist to Database
         try:
             new_analysis = AnalysisResult(

syscred/static/index.html

@@ -927,9 +927,35 @@
 
     <script>
         // Backend URLs
+        // Empty string means relative path (same domain), which works for HF Space
+        const REMOTE_API_URL = ''; 
         const LOCAL_API_URL = 'http://localhost:5001';
-        const REMOTE_API_URL = 'https://domloyer-syscred.hf.space';
-        let API_URL = '';
+        
+        // Detect if we are already on localhost
+        const isLocalhost = window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1';
+        let API_URL = isLocalhost ? LOCAL_API_URL : REMOTE_API_URL;
+        
+        // Set initial toggle state based on environment
+        document.addEventListener('DOMContentLoaded', () => {
+            const toggle = document.getElementById('backendToggle');
+            const status = document.getElementById('backendStatus');
+            const labelLocal = document.getElementById('labelLocal');
+            const labelRemote = document.getElementById('labelRemote');
+            
+            if (isLocalhost) {
+                toggle.checked = false;
+                status.textContent = 'Backend: localhost:5001 (léger, sans ML)';
+                status.className = 'backend-status local';
+                labelLocal.classList.add('active');
+                labelRemote.classList.remove('active');
+            } else {
+                toggle.checked = true;
+                status.textContent = 'Backend: HF Space (ML complet)';
+                status.className = 'backend-status remote';
+                labelLocal.classList.remove('active');
+                labelRemote.classList.add('active');
+            }
+        });
         
         function toggleBackend() {
             const toggle = document.getElementById('backendToggle');
@@ -939,7 +965,7 @@
             
             if (toggle.checked) {
                 API_URL = REMOTE_API_URL;
-                status.textContent = 'Backend: HF Space (ML complet, plus lent)';
+                status.textContent = 'Backend: HF Space (ML complet)';
                 status.className = 'backend-status remote';
                 labelLocal.classList.remove('active');
                 labelRemote.classList.add('active');
@@ -950,7 +976,7 @@
                 labelLocal.classList.add('active');
                 labelRemote.classList.remove('active');
             }
-            console.log('[SysCRED] Backend switched to:', API_URL);
+            console.log('[SysCRED] Backend switched to:', API_URL || 'Relative Path (HF Space)');
         }
 
         async function analyzeUrl() {
@@ -1158,9 +1184,23 @@
             // ========================================
             const nerSection = document.getElementById('nerSection');
             const nerEntities = document.getElementById('nerEntities');
-            const entities = nlpAnalysis.entities || data.ner_entities || [];
+            let nerData = data.ner_entities || nlpAnalysis.entities || {};
             
-            if (entities && entities.length > 0) {
+            // Convert dict format to array if needed
+            let entities = [];
+            if (Array.isArray(nerData)) {
+                entities = nerData;
+            } else if (typeof nerData === 'object' && nerData !== null) {
+                for (const [label, items] of Object.entries(nerData)) {
+                    if (Array.isArray(items)) {
+                        items.forEach(item => {
+                            entities.push({...item, label: label});
+                        });
+                    }
+                }
+            }
+            
+            if (entities.length > 0) {
                 nerSection.style.display = 'block';
                 let nerHTML = '';
                 
@@ -1204,7 +1244,7 @@
             // DISPLAY E-E-A-T METRICS
             // ========================================
             const eeatSection = document.getElementById('eeatSection');
-            const eeatData = data.eeat_score || data.eeatMetrics || null;
+            const eeatData = data.eeat_scores || data.eeat_scores || null;
             
             if (eeatData) {
                 eeatSection.style.display = 'block';
@@ -1622,18 +1662,22 @@
             
             if(!overlay) return;
             
-            title.textContent = d.name;
+            title.textContent = d.name || d.label || 'Unknown';
             
             let typeColor = "#94a3b8";
             if(d.group === 1) typeColor = "#8b5cf6"; // Report
             if(d.group === 3) typeColor = "#22c55e"; // Good
             if(d.group === 4) typeColor = "#ef4444"; // Bad
             
+            // Use uri field if available, fallback to id
+            const displayUri = d.uri || d.id || 'N/A';
+            
             body.innerHTML = `
                 <div style="margin-bottom:0.5rem">
                     <span style="background:${typeColor}; color:white; padding:2px 6px; border-radius:4px; font-size:0.75rem;">${d.type || 'Unknown Type'}</span>
                 </div>
-                <div><strong>URI:</strong> <br><span style="font-family:monospace; color:#a855f7; word-break:break-all;">${d.id}</span></div>
+                <div><strong>URI:</strong> <br><span style="font-family:monospace; color:#a855f7; word-break:break-all;">${displayUri}</span></div>
+                ${d.score ? `<div style="margin-top:0.5rem"><strong>Score:</strong> ${(d.score * 100).toFixed(0)}%</div>` : ''}
             `;
             
             overlay.classList.add('visible');