app.py

from flask import Flask, request, jsonify, render_template
from urllib.request import Request, urlopen
from bs4 import BeautifulSoup
import nltk
import re
import socket
from urllib.parse import urlparse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans
import numpy as np

# Ensure NLTK data exists
nltk.download("punkt", quiet=True)
nltk.download("punkt_tab", quiet=True)
from nltk.tokenize import word_tokenize, sent_tokenize

app = Flask(__name__)

# -------------------------
# Helper: fetch page safely
# -------------------------
def fetch_page(url, timeout=15):
    """
    Fetch URL content using urllib with a browser-like User-Agent.
    Returns cleaned text or raises Exception.
    """
    try:
        req = Request(
            url,
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                              "AppleWebKit/537.36 (KHTML, like Gecko) "
                              "Chrome/120.0 Safari/537.36"
            },
        )
        resp = urlopen(req, timeout=timeout)
        raw = resp.read()
        soup = BeautifulSoup(raw, "html.parser")

        # remove scripts/styles etc
        for tag in soup(["script", "style", "noscript", "iframe", "header", "footer"]):
            tag.extract()

        text = soup.get_text(separator=" ")
        text = re.sub(r"\s+", " ", text).strip()
        return text
    except Exception as e:
        raise

# -------------------------
# Helper: extract heading tag text
# -------------------------
def extract_heading_text(soup, tag):
    elements = soup.find_all(tag)
    return " ".join([el.get_text(" ", strip=True) for el in elements]).strip()

# -------------------------
# Clean / normalize text
# -------------------------
def clean_text(t):
    return re.sub(r"\s+", " ", t or "").strip()

# -------------------------
# Summarize (extractive)
# -------------------------
def summarize(text, num_sentences=3):
    sentences = sent_tokenize(text)
    if len(sentences) <= num_sentences:
        return " ".join(sentences)
    try:
        vec = TfidfVectorizer(stop_words="english")
        X = vec.fit_transform(sentences)
        scores = np.array(X.sum(axis=1)).ravel()
        top_idx = scores.argsort()[-num_sentences:][::-1]
        top_sentences = [sentences[i] for i in sorted(top_idx)]
        return " ".join(top_sentences)
    except Exception:
        return " ".join(sentences[:num_sentences])

# -------------------------
# Topic clustering
# -------------------------
def cluster_texts(texts, n_clusters=3):
    if len(texts) == 0:
        return []
    if len(texts) <= 1:
        return [0] * len(texts)
    k = min(n_clusters, len(texts))
    vec = TfidfVectorizer(stop_words="english")
    X = vec.fit_transform(texts)
    kmeans = KMeans(n_clusters=k, random_state=0, n_init=10)
    labels = kmeans.fit_predict(X)
    return labels.tolist()

# -------------------------
# Duplicate detection (cosine)
# -------------------------
def detect_duplicates(texts, threshold=0.55):
    n = len(texts)
    if n <= 1:
        return []
    vec = TfidfVectorizer(stop_words="english")
    X = vec.fit_transform(texts)
    sim = cosine_similarity(X)
    groups = []
    used = set()
    for i in range(n):
        if i in used:
            continue
        group = [i]
        used.add(i)
        for j in range(i + 1, n):
            if sim[i, j] >= threshold:
                group.append(j)
                used.add(j)
        if len(group) > 1:
            groups.append(group)
    return groups

# -------------------------
# Sentence-level change detection (exact-match)
# -------------------------
def changed_sentences(textA, textB):
    sA = [s.strip() for s in sent_tokenize(textA) if s.strip()]
    sB = [s.strip() for s in sent_tokenize(textB) if s.strip()]
    setA = set(sA)
    setB = set(sB)
    changedA = [s for s in sA if s not in setB]
    changedB = [s for s in sB if s not in setA]
    return changedA, changedB

# -------------------------
# Return hostname helper
# -------------------------
def hostname(url):
    try:
        p = urlparse(url)
        return p.netloc or url
    except Exception:
        return url

# -------------------------
# Routes
# -------------------------
@app.route("/")
def home():
    # list of preselected sites (you can add/remove)
    sites = {
        "Indian Express": "https://indianexpress.com/",
        "Times of India": "https://timesofindia.indiatimes.com/",
        "NDTV": "https://www.ndtv.com/",
        "BBC News": "https://www.bbc.com/news",
        "CNN": "https://www.cnn.com/",
        "The Hindu": "https://www.thehindu.com/",
    }
    return render_template("index.html", sites=sites)

@app.route("/process_urls", methods=["POST"])
def process_urls():
    payload = request.get_json(force=True)
    urls = payload.get("urls", []) or []
    mode = payload.get("mode", "tokenize")

    results = []
    texts_for_clustering = []

    for raw_url in urls:
        url = raw_url.strip()
        if not url:
            continue
        try:
            # fetch page raw
            req = Request(
                url,
                headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                                       "AppleWebKit/537.36 (KHTML, like Gecko) "
                                       "Chrome/120.0 Safari/537.36"}
            )
            resp = urlopen(req, timeout=15)
            soup = BeautifulSoup(resp.read(), "html.parser")

            # choose extraction according to mode (H1..H6 or full)
            if mode in ["H1", "H2", "H3", "H4", "H5", "H6"]:
                tag = mode.lower()
                extracted = extract_heading_text(soup, tag)
            else:
                # full text
                for tag_rm in soup(["script", "style", "noscript", "iframe", "header", "footer"]):
                    tag_rm.extract()
                extracted = soup.get_text(separator=" ")
                extracted = clean_text(extracted)

            words = []
            sentences = []
            if extracted:
                # tokenization may throw in weird content, guard it
                try:
                    words = word_tokenize(extracted)
                except Exception:
                    words = extracted.split()
                try:
                    sentences = sent_tokenize(extracted)
                except Exception:
                    sentences = [s.strip() for s in re.split(r'(?<=[.!?]) +', extracted) if s.strip()]

            summary = summarize(extracted) if extracted else ""
            texts_for_clustering.append(extracted)

            results.append({
                "url": url,
                "host": hostname(url),
                "text": extracted,
                "words": words,
                "sentences": sentences,
                "summary": summary,
            })
        except Exception as e:
            results.append({
                "url": url,
                "host": hostname(url),
                "text": "",
                "words": [],
                "sentences": [],
                "summary": "",
                "error": str(e)
            })

    # clustering
    texts_only = [r.get("text", "") for r in results]
    clusters = cluster_texts(texts_only, n_clusters=3) if len(texts_only) > 0 else []
    # attach clusters (fill default 0 if sizes mismatch)
    if len(clusters) != len(results):
        clusters = [int(c) if i < len(clusters) else 0 for i, c in enumerate(range(len(results)))]
    for i, r in enumerate(results):
        r["cluster"] = int(clusters[i]) if i < len(clusters) else 0

    # duplicate groups (convert index groups to url groups)
    dup_idx_groups = detect_duplicates(texts_only, threshold=0.55)
    dup_url_groups = [[results[i]["url"] for i in grp] for grp in dup_idx_groups]

    return jsonify({
        "articles": results,
        "duplicate_groups": dup_url_groups
    })

@app.route("/compare_texts", methods=["POST"])
def compare_texts_route():
    data = request.get_json(force=True)
    text1 = data.get("text1", "") or ""
    text2 = data.get("text2", "") or ""

    # compute changed sentences (exact-match)
    changedA, changedB = changed_sentences(text1, text2)

    # build html: show only changed sentences highlighted, and keep order from original
    def highlight_html(original_text, changed_set):
        sents = [s.strip() for s in sent_tokenize(original_text) if s.strip()]
        pieces = []
        for s in sents:
            if s in changed_set:
                pieces.append(f"<p class='changed'>{escape_html(s)}</p>")
        return "".join(pieces)

    left_html = highlight_html(text1, set(changedA))
    right_html = highlight_html(text2, set(changedB))

    return jsonify({"left": left_html, "right": right_html, "changedA_count": len(changedA), "changedB_count": len(changedB)})

# small helper used in templates/JS if needed
def escape_html(s):
    return (s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
            .replace('"', "&quot;").replace("'", "&#39;"))

if __name__ == "__main__":
    # increase default socket timeout a bit
    socket.setdefaulttimeout(20)
    app.run(host="0.0.0.0", port=7860, debug=False)