Spaces:

eriksarriegui
/

news-intelligence

Running

App Files Files Community

Erik Sarriegui commited on Feb 8

Commit

588462f

1 Parent(s): d3c610f

first push

Browse files

Files changed (16) hide show

Dockerfile +44 -0
frontend/index.html +16 -0
frontend/package.json +29 -0
frontend/postcss.config.js +6 -0
frontend/src/App.jsx +78 -0
frontend/src/components/Aggregator.jsx +108 -0
frontend/src/components/Analyzer.jsx +141 -0
frontend/src/components/ClusterCard.jsx +78 -0
frontend/src/index.css +9 -0
frontend/src/lib/utils.js +6 -0
frontend/src/main.jsx +10 -0
frontend/tailwind.config.js +22 -0
frontend/vite.config.js +21 -0
main.py +196 -0
requirements.txt +12 -0
src/dataset.parquet +3 -0

Dockerfile ADDED Viewed

	@@ -0,0 +1,44 @@

+# Stage 1: Build Frontend
+FROM node:18-alpine as frontend-build
+WORKDIR /app/frontend
+COPY frontend/package.json ./
+RUN npm install
+COPY frontend/ ./
+RUN npm run build
+# Stage 2: Backend & Runtime
+FROM python:3.10-slim
+WORKDIR /app
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libxml2-dev \
+    libxslt-dev \
+    && rm -rf /var/lib/apt/lists/*
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+# Copy backend code
+COPY main.py .
+COPY src/ src/
+# Copy built frontend from Stage 1
+COPY --from=frontend-build /app/frontend/dist /app/static
+# Create cache directory for Hugging Face models to ensure they are writable
+# HF Spaces run as user 1000
+RUN mkdir -p /app/cache && chmod 777 /app/cache
+ENV HF_HOME=/app/cache
+# Expose port (7860 is default for HF Spaces)
+EXPOSE 7860
+# Run the app
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

frontend/index.html ADDED Viewed

	@@ -0,0 +1,16 @@

+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>News Intelligence</title>
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap" rel="stylesheet">
+  </head>
+  <body class="bg-slate-950 text-slate-50 font-sans">
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

frontend/package.json ADDED Viewed

	@@ -0,0 +1,29 @@

+{
+    "name": "frontend",
+    "private": true,
+    "version": "0.0.0",
+    "type": "module",
+    "scripts": {
+        "dev": "vite",
+        "build": "vite build",
+        "preview": "vite preview"
+    },
+    "dependencies": {
+        "axios": "^1.6.7",
+        "clsx": "^2.1.0",
+        "framer-motion": "^11.0.3",
+        "lucide-react": "^0.330.0",
+        "react": "^18.2.0",
+        "react-dom": "^18.2.0",
+        "tailwind-merge": "^2.2.1"
+    },
+    "devDependencies": {
+        "@types/react": "^18.2.55",
+        "@types/react-dom": "^18.2.19",
+        "@vitejs/plugin-react": "^4.2.1",
+        "autoprefixer": "^10.4.17",
+        "postcss": "^8.4.35",
+        "tailwindcss": "^3.4.1",
+        "vite": "^5.1.0"
+    }
+}

frontend/postcss.config.js ADDED Viewed

	@@ -0,0 +1,6 @@

+export default {
+    plugins: {
+        tailwindcss: {},
+        autoprefixer: {},
+    },
+}

frontend/src/App.jsx ADDED Viewed

	@@ -0,0 +1,78 @@

+import { useState } from 'react';
+import { Newspaper, Search, LayoutGrid } from 'lucide-react';
+import { motion, AnimatePresence } from 'framer-motion';
+import Aggregator from './components/Aggregator';
+import Analyzer from './components/Analyzer';
+import { cn } from './lib/utils';
+function App() {
+    const [activeTab, setActiveTab] = useState('aggregator');
+    return (
+        <div className="min-h-screen bg-slate-950 text-slate-100 flex flex-col items-center py-10 px-4">
+            <header className="mb-8 text-center">
+                <h1 className="text-4xl font-bold bg-gradient-to-r from-blue-400 to-purple-500 bg-clip-text text-transparent mb-2">
+                    News Intelligence
+                </h1>
+                <p className="text-slate-400">Advanced Aggregation & Analysis</p>
+            </header>
+            {/* Navigation */}
+            <div className="bg-slate-900/50 p-1 rounded-xl flex gap-1 mb-8 border border-slate-800 backdrop-blur-sm">
+                <button
+                    onClick={() => setActiveTab('aggregator')}
+                    className={cn(
+                        "flex items-center gap-2 px-6 py-2.5 rounded-lg text-sm font-medium transition-all duration-300",
+                        activeTab === 'aggregator'
+                            ? "bg-blue-600 text-white shadow-lg shadow-blue-900/20"
+                            : "text-slate-400 hover:text-white hover:bg-slate-800"
+                    )}
+                >
+                    <LayoutGrid className="w-4 h-4" />
+                    Event Stream
+                </button>
+                <button
+                    onClick={() => setActiveTab('analyzer')}
+                    className={cn(
+                        "flex items-center gap-2 px-6 py-2.5 rounded-lg text-sm font-medium transition-all duration-300",
+                        activeTab === 'analyzer'
+                            ? "bg-purple-600 text-white shadow-lg shadow-purple-900/20"
+                            : "text-slate-400 hover:text-white hover:bg-slate-800"
+                    )}
+                >
+                    <Search className="w-4 h-4" />
+                    Analyzer
+                </button>
+            </div>
+            {/* Content Area */}
+            <main className="w-full max-w-5xl">
+                <AnimatePresence mode="wait">
+                    {activeTab === 'aggregator' ? (
+                        <motion.div
+                            key="aggregator"
+                            initial={{ opacity: 0, y: 20 }}
+                            animate={{ opacity: 1, y: 0 }}
+                            exit={{ opacity: 0, y: -20 }}
+                            transition={{ duration: 0.3 }}
+                        >
+                            <Aggregator />
+                        </motion.div>
+                    ) : (
+                        <motion.div
+                            key="analyzer"
+                            initial={{ opacity: 0, y: 20 }}
+                            animate={{ opacity: 1, y: 0 }}
+                            exit={{ opacity: 0, y: -20 }}
+                            transition={{ duration: 0.3 }}
+                        >
+                            <Analyzer />
+                        </motion.div>
+                    )}
+                </AnimatePresence>
+            </main>
+        </div>
+    );
+}
+export default App;

frontend/src/components/Aggregator.jsx ADDED Viewed

	@@ -0,0 +1,108 @@

+import { useState, useEffect } from 'react';
+import axios from 'axios';
+import { Calendar, RefreshCw } from 'lucide-react';
+import ClusterCard from './ClusterCard';
+import { cn } from '../lib/utils';
+import { motion } from 'framer-motion';
+export default function Aggregator() {
+    const [data, setData] = useState(null);
+    const [loading, setLoading] = useState(true);
+    const [error, setError] = useState(null);
+    const [selectedDayIndex, setSelectedDayIndex] = useState(0);
+    useEffect(() => {
+        fetchData();
+    }, []);
+    const fetchData = async () => {
+        try {
+            setLoading(true);
+            setError(null);
+            const response = await axios.get('/api/clusters');
+            // Sort daysdescending just in case or keep as valid from API
+            // API returns top 3 days. We'll use them as is.
+            setData(response.data);
+        } catch (err) {
+            console.error(err);
+            setError("Failed to load clusters. Please try again.");
+        } finally {
+            setLoading(false);
+        }
+    };
+    if (loading) {
+        return (
+            <div className="flex flex-col items-center justify-center py-20 text-slate-500">
+                <RefreshCw className="w-8 h-8 animate-spin mb-4 text-blue-500" />
+                <p>Loading intelligence stream...</p>
+            </div>
+        );
+    }
+    if (error) {
+        return (
+            <div className="text-center py-20 text-red-400 bg-red-900/10 rounded-xl border border-red-900/20">
+                <p className="mb-4">{error}</p>
+                <button
+                    onClick={fetchData}
+                    className="px-4 py-2 bg-red-600 text-white rounded-lg hover:bg-red-500 transition-colors"
+                >
+                    Retry
+                </button>
+            </div>
+        );
+    }
+    if (!data || data.length === 0) {
+        return <div className="text-center py-20 text-slate-500">No data available.</div>;
+    }
+    const currentDay = data[selectedDayIndex];
+    return (
+        <div className="space-y-6">
+            {/* Day Tabs */}
+            <div className="flex justify-center gap-4">
+                {data.map((dayItem, index) => (
+                    <button
+                        key={dayItem.date}
+                        onClick={() => setSelectedDayIndex(index)}
+                        className={cn(
+                            "flex flex-col items-center p-3 rounded-xl border transition-all duration-300 w-32",
+                            selectedDayIndex === index
+                                ? "bg-slate-800 border-blue-500/50 shadow-lg shadow-blue-500/10 text-white"
+                                : "bg-slate-900/50 border-slate-800 text-slate-400 hover:bg-slate-800 hover:border-slate-700"
+                        )}
+                    >
+                        <span className="text-xs font-medium uppercase tracking-wider text-slate-500 mb-1">
+                            {new Date(dayItem.date).toLocaleDateString(undefined, { weekday: 'short' })}
+                        </span>
+                        <span className="text-lg font-bold">
+                            {new Date(dayItem.date).toLocaleDateString(undefined, { day: '2-digit', month: 'short' })}
+                        </span>
+                    </button>
+                ))}
+            </div>
+            {/* Stats for the day */}
+            <div className="flex items-center justify-between px-4 pb-2 text-slate-400 text-sm border-b border-slate-800">
+                <div className="flex items-center gap-2">
+                    <Calendar className="w-4 h-4" />
+                    <span>Events for {new Date(currentDay.date).toLocaleDateString()}</span>
+                </div>
+                <div>
+                    {currentDay.clusters.length} Event Clusters
+                </div>
+            </div>
+            {/* Clusters Grid */}
+            <div className="grid grid-cols-1 md:grid-cols-1 gap-4">
+                {currentDay.clusters.map((cluster) => (
+                    <ClusterCard key={cluster.cluster_id} cluster={cluster} />
+                ))}
+            </div>
+        </div>
+    );
+}

frontend/src/components/Analyzer.jsx ADDED Viewed

	@@ -0,0 +1,141 @@

+import { useState } from 'react';
+import axios from 'axios';
+import { Search, AlertTriangle, CheckCircle, AlertOctagon, ArrowRight, Loader2 } from 'lucide-react';
+import { motion } from 'framer-motion';
+import { cn } from '../lib/utils';
+export default function Analyzer() {
+    const [url, setUrl] = useState('');
+    const [result, setResult] = useState(null);
+    const [loading, setLoading] = useState(false);
+    const [error, setError] = useState(null);
+    const handleAnalyze = async (e) => {
+        e.preventDefault();
+        if (!url) return;
+        setLoading(true);
+        setError(null);
+        setResult(null);
+        try {
+            const response = await axios.post('/api/analyze', { url });
+            setResult(response.data);
+        } catch (err) {
+            console.error(err);
+            setError("Analysis failed. Please check the URL or try again later.");
+        } finally {
+            setLoading(false);
+        }
+    };
+    return (
+        <div className="max-w-3xl mx-auto space-y-8">
+            {/* Input Section */}
+            <div className="bg-slate-900/50 p-8 rounded-2xl border border-slate-800 shadow-xl backdrop-blur-sm">
+                <h2 className="text-2xl font-bold mb-6 text-center text-slate-100">Analyze News Article</h2>
+                <form onSubmit={handleAnalyze} className="flex gap-2">
+                    <div className="relative flex-1">
+                        <Search className="absolute left-4 top-1/2 -translate-y-1/2 text-slate-500 w-5 h-5" />
+                        <input
+                            type="url"
+                            placeholder="Paste article URL here (e.g., https://elpais.com/...)"
+                            value={url}
+                            onChange={(e) => setUrl(e.target.value)}
+                            className="w-full bg-slate-950 border border-slate-700 text-slate-100 pl-11 pr-4 py-4 rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500/50 transition-all placeholder:text-slate-600"
+                            required
+                        />
+                    </div>
+                    <button
+                        type="submit"
+                        disabled={loading}
+                        className="bg-blue-600 hover:bg-blue-500 text-white px-8 rounded-xl font-medium transition-colors disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2"
+                    >
+                        {loading ? <Loader2 className="animate-spin w-5 h-5" /> : "Analyze"}
+                    </button>
+                </form>
+            </div>
+            {/* Error Message */}
+            {error && (
+                <motion.div
+                    initial={{ opacity: 0, y: 10 }}
+                    animate={{ opacity: 1, y: 0 }}
+                    className="p-4 bg-red-950/30 border border-red-900/50 rounded-xl text-red-400 text-center"
+                >
+                    {error}
+                </motion.div>
+            )}
+            {/* Results Section */}
+            {result && (
+                <motion.div
+                    initial={{ opacity: 0, scale: 0.95 }}
+                    animate={{ opacity: 1, scale: 1 }}
+                    className="bg-slate-900/50 rounded-2xl border border-slate-800 overflow-hidden shadow-2xl"
+                >
+                    {/* Header */}
+                    <div className="p-6 border-b border-slate-800 bg-slate-900/80">
+                        <h3 className="text-xl font-semibold text-slate-100 leading-snug">{result.title}</h3>
+                        <p className="mt-2 text-slate-400 text-sm line-clamp-3 leading-relaxed">
+                            {result.text_snippet}
+                        </p>
+                    </div>
+                    {/* Analysis Cards */}
+                    <div className="grid grid-cols-1 md:grid-cols-2 gap-px bg-slate-800">
+                        {/* Clickbait Result */}
+                        <div className={cn(
+                            "p-8 flex flex-col items-center justify-center text-center gap-4",
+                            result.is_clickbait ? "bg-red-950/20" : "bg-emerald-950/20"
+                        )}>
+                            <div className={cn(
+                                "w-16 h-16 rounded-full flex items-center justify-center mb-2 shadow-lg",
+                                result.is_clickbait ? "bg-orange-500/20 text-orange-500" : "bg-emerald-500/20 text-emerald-500"
+                            )}>
+                                {result.is_clickbait ? <AlertTriangle className="w-8 h-8" /> : <CheckCircle className="w-8 h-8" />}
+                            </div>
+                            <div>
+                                <h4 className="text-slate-400 text-sm font-medium uppercase tracking-widest mb-1">Headline Analysis</h4>
+                                <p className={cn(
+                                    "text-2xl font-bold",
+                                    result.is_clickbait ? "text-orange-400" : "text-emerald-400"
+                                )}>
+                                    {result.is_clickbait ? "Clickbait Detected" : "No Clickbait"}
+                                </p>
+                            </div>
+                            <div className="text-xs text-slate-500 bg-slate-900/50 px-3 py-1 rounded-full">
+                                Confidence: {(result.clickbait_conf * 100).toFixed(1)}%
+                            </div>
+                        </div>
+                        {/* Sensationalism Result */}
+                        <div className={cn(
+                            "p-8 flex flex-col items-center justify-center text-center gap-4",
+                            result.is_sensationalist ? "bg-red-950/20" : "bg-emerald-950/20"
+                        )}>
+                            <div className={cn(
+                                "w-16 h-16 rounded-full flex items-center justify-center mb-2 shadow-lg",
+                                result.is_sensationalist ? "bg-red-500/20 text-red-500" : "bg-blue-500/20 text-blue-500"
+                            )}>
+                                {result.is_sensationalist ? <AlertOctagon className="w-8 h-8" /> : <CheckCircle className="w-8 h-8" />}
+                            </div>
+                            <div>
+                                <h4 className="text-slate-400 text-sm font-medium uppercase tracking-widest mb-1">Tone Analysis</h4>
+                                <p className={cn(
+                                    "text-2xl font-bold",
+                                    result.is_sensationalist ? "text-red-400" : "text-blue-400"
+                                )}>
+                                    {result.is_sensationalist ? "Sensationalist" : "Neutral Tone"}
+                                </p>
+                            </div>
+                            <div className="text-xs text-slate-500 bg-slate-900/50 px-3 py-1 rounded-full">
+                                Confidence: {(result.sensationalist_conf * 100).toFixed(1)}%
+                            </div>
+                        </div>
+                    </div>
+                </motion.div>
+            )}
+        </div>
+    );
+}

frontend/src/components/ClusterCard.jsx ADDED Viewed

	@@ -0,0 +1,78 @@

+import { useState } from 'react';
+import { motion, AnimatePresence } from 'framer-motion';
+import { ChevronDown, ExternalLink, Hash, FileText } from 'lucide-react';
+import { cn } from '../lib/utils';
+export default function ClusterCard({ cluster }) {
+    const [isExpanded, setIsExpanded] = useState(false);
+    return (
+        <motion.div
+            layout
+            className="bg-slate-900/50 border border-slate-800 rounded-xl overflow-hidden hover:border-slate-700 transition-colors"
+        >
+            <div
+                onClick={() => setIsExpanded(!isExpanded)}
+                className="p-5 cursor-pointer flex items-start justify-between gap-4"
+            >
+                <div className="flex-1">
+                    <div className="flex items-center gap-2 mb-2">
+                        <span className="bg-blue-500/10 text-blue-400 text-xs px-2 py-0.5 rounded-full border border-blue-500/20 font-medium flex items-center gap-1">
+                            <Hash className="w-3 h-3" />
+                            Cluster {cluster.cluster_id}
+                        </span>
+                        <span className="bg-slate-800 text-slate-400 text-xs px-2 py-0.5 rounded-full font-medium flex items-center gap-1">
+                            <FileText className="w-3 h-3" />
+                            {cluster.size} articles
+                        </span>
+                    </div>
+                    <h3 className="text-lg font-semibold text-slate-200 leading-tight group-hover:text-blue-400 transition-colors">
+                        {cluster.title}
+                    </h3>
+                </div>
+                <motion.div
+                    animate={{ rotate: isExpanded ? 180 : 0 }}
+                    transition={{ duration: 0.2 }}
+                    className="bg-slate-800 p-1.5 rounded-lg text-slate-400"
+                >
+                    <ChevronDown className="w-5 h-5" />
+                </motion.div>
+            </div>
+            <AnimatePresence>
+                {isExpanded && (
+                    <motion.div
+                        initial={{ height: 0, opacity: 0 }}
+                        animate={{ height: "auto", opacity: 1 }}
+                        exit={{ height: 0, opacity: 0 }}
+                        className="border-t border-slate-800 bg-slate-900/30"
+                    >
+                        <div className="p-4 space-y-3">
+                            {cluster.articles.map((article, idx) => (
+                                <a
+                                    key={idx}
+                                    href={article.url}
+                                    target="_blank"
+                                    rel="noopener noreferrer"
+                                    className="block p-3 rounded-lg bg-slate-800/40 hover:bg-slate-800 transition-colors group"
+                                >
+                                    <div className="flex justify-between items-start gap-3">
+                                        <h4 className="text-sm text-slate-300 font-medium line-clamp-2 group-hover:text-blue-300">
+                                            {article.title}
+                                        </h4>
+                                        <ExternalLink className="w-4 h-4 text-slate-500 shrink-0 group-hover:text-blue-400" />
+                                    </div>
+                                    <div className="mt-2 flex items-center gap-2 text-xs text-slate-500">
+                                        <span className="font-semibold text-slate-400">{article.newspaper}</span>
+                                        <span>•</span>
+                                        <span className="truncate max-w-[200px]">{article.newspaper_url}</span>
+                                    </div>
+                                </a>
+                            ))}
+                        </div>
+                    </motion.div>
+                )}
+            </AnimatePresence>
+        </motion.div>
+    );
+}

frontend/src/index.css ADDED Viewed

	@@ -0,0 +1,9 @@

+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+@layer base {
+  body {
+    @apply bg-slate-950 text-slate-100;
+  }
+}

frontend/src/lib/utils.js ADDED Viewed

	@@ -0,0 +1,6 @@

+import { clsx } from "clsx"
+import { twMerge } from "tailwind-merge"
+export function cn(...inputs) {
+    return twMerge(clsx(inputs))
+}

frontend/src/main.jsx ADDED Viewed

	@@ -0,0 +1,10 @@

+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App.jsx'
+import './index.css'
+ReactDOM.createRoot(document.getElementById('root')).render(
+    <React.StrictMode>
+        <App />
+    </React.StrictMode>,
+)

frontend/tailwind.config.js ADDED Viewed

	@@ -0,0 +1,22 @@

+/** @type {import('tailwindcss').Config} */
+export default {
+    content: [
+        "./index.html",
+        "./src/**/*.{js,ts,jsx,tsx}",
+    ],
+    theme: {
+        extend: {
+            fontFamily: {
+                sans: ['Inter', 'sans-serif'],
+            },
+            colors: {
+                primary: "#2563EB",
+                secondary: "#475569",
+                accent: "#F59E0B",
+                background: "#0F172A",
+                surface: "#1E293B",
+            }
+        },
+    },
+    plugins: [],
+}

frontend/vite.config.js ADDED Viewed

	@@ -0,0 +1,21 @@

+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react'
+import path from 'path'
+// https://vitejs.dev/config/
+export default defineConfig({
+    plugins: [react()],
+    resolve: {
+        alias: {
+            '@': path.resolve(__dirname, './src'),
+        },
+    },
+    server: {
+        proxy: {
+            '/api': {
+                target: 'http://localhost:8000',
+                changeOrigin: true,
+            },
+        },
+    },
+})

main.py ADDED Viewed

	@@ -0,0 +1,196 @@

+import os
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+from typing import List, Dict, Any
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+import newspaper
+from newspaper import Article
+import logging
+import nltk
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+app = FastAPI()
+# CORS configuration
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+# Global variables
+DATA_PATH = "src/dataset.parquet"
+df = None
+models = {}
+# Load Data
+def load_data():
+    global df
+    try:
+        logger.info(f"Loading data from {DATA_PATH}...")
+        df = pd.read_parquet(DATA_PATH)
+        # Ensure date is string or datetime for consistent handling
+        df['date'] = df['date'].astype(str)
+        logger.info("Data loaded successfully.")
+    except Exception as e:
+        logger.error(f"Error loading data: {e}")
+        df = pd.DataFrame()
+# Load Models
+def load_models():
+    # Only load if not already loaded to save resources on reload (though in docker it's once)
+    if "clickbait" in models:
+        return
+    try:
+        logger.info("Loading Clickbait Model...")
+        cb_model_name = "eriksarriegui/mmBERT-base-clickbait-detection-es"
+        models["cb_tokenizer"] = AutoTokenizer.from_pretrained(cb_model_name)
+        models["cb_model"] = AutoModelForSequenceClassification.from_pretrained(cb_model_name)
+        logger.info("Loading Sensationalism Model...")
+        sens_model_name = "eriksarriegui/mmBERT-base-sensacionalism-detection-es"
+        models["sens_tokenizer"] = AutoTokenizer.from_pretrained(sens_model_name)
+        models["sens_model"] = AutoModelForSequenceClassification.from_pretrained(sens_model_name)
+        logger.info("Models loaded successfully.")
+    except Exception as e:
+        logger.error(f"Error loading models: {e}")
+@app.on_event("startup")
+async def startup_event():
+    try:
+        nltk.download('punkt')
+        nltk.download('punkt_tab')
+    except Exception as e:
+        logger.warning(f"NLTK download warning: {e}")
+    load_data()
+    # We might want to load models lazily if memory is tight, but for now let's load on startup
+    # or trigger it on first request. Let's trigger on startup to fail fast.
+    # Note: Downloading models takes time.
+    load_models()
+# Pydantic Models
+class AnalyzeRequest(BaseModel):
+    url: str
+class ClusterResponse(BaseModel):
+    date: str
+    clusters: List[Dict[str, Any]]
+# Endpoints
+@app.get("/api/clusters")
+def get_clusters():
+    if df is None or df.empty:
+        raise HTTPException(status_code=500, detail="Data not loaded")
+    # Group by date and count clusters
+    # Assuming 'date' is YYYY-MM-DD
+    # cluster_id uniquely identifies a cluster.
+    # 1. Count clusters per day
+    # We count unique cluster_ids per date
+    daily_counts = df.groupby('date')['cluster_id'].nunique().reset_index()
+    daily_counts.columns = ['date', 'count']
+    # 2. Get top 3 days
+    top_days = daily_counts.sort_values('count', ascending=False).head(3)['date'].tolist()
+    result = []
+    for day in top_days:
+        day_data = df[df['date'] == day]
+        # Group by cluster_id
+        clusters_data = []
+        for cluster_id, group in day_data.groupby('cluster_id'):
+            # Pick a representative title (e.g., the first one or longest)
+            # We'll take the first one for simplicity, or maybe the one with most extracted_authors?
+            rep_article = group.iloc[0]
+            cluster_articles = []
+            for _, row in group.iterrows():
+                cluster_articles.append({
+                    "title": row.get('title', 'No Title'),
+                    "newspaper": row.get('newspaper', 'Unknown'),
+                    "url": row.get('article_url', '#'),
+                    "newspaper_url": row.get('newspaper_url', '#')
+                })
+            clusters_data.append({
+                "cluster_id": str(cluster_id),
+                "title": rep_article.get('title', 'Cluster Title'),
+                "size": int(group['cluster_size'].iloc[0]) if 'cluster_size' in group else len(group),
+                "articles": cluster_articles
+            })
+        result.append({
+            "date": day,
+            "clusters": clusters_data
+        })
+    return result
+@app.post("/api/analyze")
+def analyze_article(request: AnalyzeRequest):
+    url = request.url
+    try:
+        # Scrape
+        article = Article(url)
+        article.download()
+        article.parse()
+        title = article.title
+        text = article.text
+        # Inference
+        # Clickbait: using title
+        cb_inputs = models["cb_tokenizer"](title, return_tensors="pt", truncation=True, max_length=512)
+        with torch.no_grad():
+            cb_outputs = models["cb_model"](**cb_inputs)
+        cb_probs = torch.softmax(cb_outputs.logits, dim=1)
+        # Assuming label 0 is NO, 1 is YES (need to verify mapping usually, usually 1 is positive class)
+        # Checking model config locally or on HF would be ideal. Let's assume standard mapping:
+        # For clickbait: likely labels are "no_clickbait", "clickbait".
+        # We can check specific model details if needed. For now assume:
+        # Label 1 is clickbait.
+        is_clickbait = torch.argmax(cb_probs).item() == 1
+        # Sensationalism: using title + text
+        sens_input_text = f"TITULAR: {title}\nCUERPO: {text}"
+        sens_inputs = models["sens_tokenizer"](sens_input_text, return_tensors="pt", truncation=True, max_length=512)
+        with torch.no_grad():
+            sens_outputs = models["sens_model"](**sens_inputs)
+        sens_probs = torch.softmax(sens_outputs.logits, dim=1)
+        is_sensationalist = torch.argmax(sens_probs).item() == 1
+        return {
+            "title": title,
+            "text_snippet": text[:200] + "..." if len(text) > 200 else text,
+            "is_clickbait": is_clickbait,
+            "is_sensationalist": is_sensationalist,
+            "clickbait_conf": float(cb_probs[0][1]),
+            "sensationalist_conf": float(sens_probs[0][1])
+        }
+    except Exception as e:
+        logger.error(f"Analysis failed: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+# Static files for Frontend (will be populated after build)
+app.mount("/", StaticFiles(directory="static", html=True), name="static")
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

requirements.txt ADDED Viewed

	@@ -0,0 +1,12 @@

+fastapi
+uvicorn
+pandas
+pyarrow
+torch
+transformers
+newspaper3k
+lxml[html_clean]
+python-multipart
+scipy
+sklearn
+emoji

src/dataset.parquet ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:eda3135ba3230dfe801b6b4cd0ac7b127a95abee7be3dd4687192911c510c834
+size 28200022