Upload app.py

app.py

@@ -0,0 +1,669 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+import time
+import os
+import re
+import html as html_lib
+from typing import List, Tuple, Union
+
+import requests
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+APP_NAME = "pr-tool-backend"
+
+VK_API_VERSION = os.getenv("VK_API_VERSION", "5.131")
+VK_ACCESS_TOKEN = os.getenv("VK_ACCESS_TOKEN", "")
+REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "15"))
+MAX_WORKERS = int(os.getenv("MAX_WORKERS", "8"))
+VK_MAX_RETRIES = int(os.getenv("VK_MAX_RETRIES", "5"))
+VK_RETRY_DELAY = float(os.getenv("VK_RETRY_DELAY", "0.45"))
+VK_BATCH_SIZE = int(os.getenv("VK_BATCH_SIZE", "100"))
+
+app = FastAPI(title=APP_NAME)
+
+# Для простоты разрешаем все источники. Можно сузить список доменов в проде.
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,
+    allow_methods=["POST", "GET", "OPTIONS"],
+    allow_headers=["*"]
+)
+
+
+class ParseRequest(BaseModel):
+    links: Union[List[str], str]
+
+
+class ParseResponse(BaseModel):
+    html: str
+    text: str
+    telegram_total: int
+    vk_total: int
+    errors: List[str]
+
+
+# ---------- Вспомогательные функции ----------
+
+def human_format_views(num: int) -> str:
+    if num >= 1_000_000:
+        value = num / 1_000_000
+        suffix = "M"
+    else:
+        value = num / 1000
+        suffix = "K"
+
+    rounded = round(value, 1)
+    if rounded.is_integer():
+        formatted = f"{int(rounded)}{suffix}"
+    else:
+        formatted = f"{rounded:.1f}{suffix}"
+
+    return formatted.replace(".", ",")
+
+
+def detect_platform(link: str) -> str:
+    link = link.strip().lower()
+    if "t.me/" in link or "telegram.me/" in link:
+        return "telegram"
+    if "vk.com/wall" in link or "vk.ru/wall" in link:
+        return "vk"
+    return ""
+
+
+def canonicalize_tg_link(link: str) -> Tuple[str | None, str | None, str | None]:
+    link = link.strip()
+    m = re.match(r"https?://(?:t(?:elegram)?\.me)/(?:s/)?([^/]+)/(?P<id>\d+)", link, re.IGNORECASE)
+    if not m:
+        return None, None, None
+    username = m.group(1)
+    message_id = m.group("id")
+    canonical = f"https://t.me/{username}/{message_id}"
+    return canonical, username, message_id
+
+
+def canonicalize_vk_link(link: str) -> Tuple[str | None, int | None, str | None]:
+    link = link.strip()
+    m = re.search(r"(?:vk\.com|vk\.ru)/wall(-?\d+)_(\d+)", link)
+    if not m:
+        return None, None, None
+    owner_id = int(m.group(1))
+    post_id = m.group(2)
+    canonical = f"https://vk.com/wall{owner_id}_{post_id}"
+    return canonical, owner_id, post_id
+
+
+def process_telegram_link(channel_username: str, message_id: str, canonical_link: str) -> Tuple[str, int]:
+    # Набор альтернативных адресов (если t.me не резолвится)
+    urls = [
+        f"https://t.me/s/{channel_username}/{message_id}",
+        f"https://t.me/{channel_username}/{message_id}?embed=1",
+        f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}?embed=1",
+        f"https://r.jina.ai/http://t.me/s/{channel_username}/{message_id}",
+    ]
+    headers = {"User-Agent": "Mozilla/5.0"}
+
+    def _parse_views_number(s: str) -> int:
+        if not s:
+            return 0
+        s = s.strip().replace("\u00a0", "").replace(" ", "").replace(",", ".")
+        m2 = re.match(r"([\d\.]+)\s*([kKmM]?)", s)
+        if not m2:
+            return 0
+        val = float(m2.group(1))
+        suf = m2.group(2).lower()
+        if suf == "k":
+            val *= 1_000
+        elif suf == "m":
+            val *= 1_000_000
+        return int(val)
+
+    def _fetch_ok_responses(url_list):
+        texts: List[str] = []
+        last_err = None
+        for u in url_list:
+            try:
+                resp = requests.get(u, headers=headers, timeout=REQUEST_TIMEOUT)
+                if resp.status_code == 200 and resp.text:
+                    texts.append(resp.text)
+            except Exception as exc:
+                last_err = exc
+        if texts:
+            return texts
+        raise last_err or Exception("Не удалось получить страницу Telegram")
+
+    def _strip_emoji(s: str) -> str:
+        # Убираем emoji/pictographs и variation selectors.
+        s = re.sub(
+            r"[\U0001F1E6-\U0001F1FF\U0001F300-\U0001FAFF\u2600-\u27BF\uFE0E\uFE0F]",
+            "",
+            s,
+        )
+        return re.sub(r"\s+", " ", s).strip()
+
+    def _clean_title(raw: str, fallback: str) -> str:
+        title = re.sub(r"<[^>]+>", "", (raw or "").strip())
+        title = html_lib.unescape(title)
+        title = re.sub(r"\s*[-|]\s*Telegram\s*$", "", title, flags=re.IGNORECASE)
+        title = _strip_emoji(title)
+        return title or fallback
+
+    def _is_placeholder_title(raw: str | None) -> bool:
+        if not raw:
+            return True
+        normalized = _clean_title(raw, "").strip().lower()
+        placeholders = {
+            "",
+            "telegram",
+            "telegram widget",
+        }
+        if normalized in placeholders:
+            return True
+        if normalized.startswith("telegram:"):
+            return True
+        return False
+
+    def _extract_views(html: str) -> int:
+        patterns = [
+            # Виджет Telegram
+            r'class="tgme_widget_message_views[^"]*">([\d\s\.,kKmM]+)<',
+            # Резервный вариант из meta-description, но только если рядом явно "views"
+            r'content="[^"]*?([\d\s\.,kKmM]+)\s+views[^"]*"',
+            r'content="[^"]*?(?:Просмотры|просмотры|просмотров)[:\s]*([\d\s\.,kKmM]+)[^"]*"',
+            # Текстовый fallback (например, через jina)
+            r'(?:^|\n)\s*(?:Views|views|Просмотры|просмотры|Просмотров|просмотров)\s*[:\-]?\s*([\d\s\.,kKmM]+)\b',
+            r'([\d\s\.,kKmM]+)\s*(?:views|Views|просмотров|Просмотров)\b',
+            r'(?:👁|👀)\s*([\d\s\.,kKmM]+)\b',
+        ]
+        for pattern in patterns:
+            m = re.search(pattern, html, re.IGNORECASE)
+            if m:
+                parsed = _parse_views_number(m.group(1))
+                if parsed > 0:
+                    return parsed
+        return 0
+
+    def _extract_text_title(raw_text: str) -> str | None:
+        patterns = [
+            r'(?:^|\n)\s*Title:\s*(.+)',
+            r'(?:^|\n)\s*Channel:\s*(.+)',
+            r'(?:^|\n)\s*#\s+(.+)',
+        ]
+        for pattern in patterns:
+            m = re.search(pattern, raw_text, re.IGNORECASE)
+            if m:
+                return m.group(1).strip()
+
+        for line in raw_text.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            if line.startswith("http://") or line.startswith("https://"):
+                continue
+            if "views" in line.lower() or "просмотр" in line.lower():
+                continue
+            if len(line) <= 120:
+                return line
+        return None
+
+    def _extract_title(raw_text: str) -> str | None:
+        candidates: List[str] = []
+
+        m_site_name = re.search(
+            r"<meta[^>]*property=[\"']og:site_name[\"'][^>]*content=[\"']([^\"']+)[\"']",
+            raw_text,
+            re.IGNORECASE,
+        )
+        if m_site_name:
+            candidates.append(m_site_name.group(1).strip())
+
+        m_title_meta = re.search(
+            r"<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
+            raw_text,
+            re.IGNORECASE,
+        )
+        if m_title_meta:
+            candidates.append(m_title_meta.group(1).strip())
+
+        m_author = re.search(
+            r'class="tgme_widget_message_author_name".*?<span[^>]*>(.*?)</span>',
+            raw_text,
+            re.IGNORECASE | re.DOTALL,
+        )
+        if m_author:
+            candidates.append(m_author.group(1).strip())
+
+        m_owner = re.search(
+            r'class="tgme_widget_message_owner_name".*?<span[^>]*>(.*?)</span>',
+            raw_text,
+            re.IGNORECASE | re.DOTALL,
+        )
+        if m_owner:
+            candidates.append(m_owner.group(1).strip())
+
+        text_title = _extract_text_title(raw_text)
+        if text_title:
+            candidates.append(text_title)
+
+        for candidate in candidates:
+            if not _is_placeholder_title(candidate):
+                return candidate
+        return None
+
+    try:
+        responses = _fetch_ok_responses(urls)
+
+        title_raw = None
+        views = 0
+        for response_text in responses:
+            candidate_title = _extract_title(response_text)
+            if candidate_title and not title_raw:
+                title_raw = candidate_title
+
+            candidate_views = _extract_views(response_text)
+            if candidate_views > views:
+                views = candidate_views
+
+            if title_raw and views > 0:
+                break
+
+        title = _clean_title(title_raw or "", channel_username)
+
+        return title, views
+
+    except Exception as exc:
+        return f"Ошибка (TG) при обработке {canonical_link}: {exc}", 0
+
+
+def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[str, int]:
+    posts_param = f"{owner_id}_{post_id}"
+    api_url = "https://api.vk.com/method/wall.getById"
+
+    for attempt in range(1, VK_MAX_RETRIES + 1):
+        params = {
+            "posts": posts_param,
+            "v": VK_API_VERSION,
+            "extended": 1,
+        }
+        if VK_ACCESS_TOKEN:
+            params["access_token"] = VK_ACCESS_TOKEN
+
+        try:
+            resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
+        except Exception as exc:
+            return f"**Ошибка (VK)**: {exc} — {canonical_link}", 0
+
+        if resp.status_code != 200:
+            return f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical_link}", 0
+
+        data = resp.json()
+        if "error" in data:
+            error = data["error"]
+            error_msg = error.get("error_msg", "")
+            error_code = error.get("error_code")
+            if error_code == 6 or "too many requests per second" in error_msg.lower():
+                time.sleep(VK_RETRY_DELAY * attempt)
+                continue
+            return f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical_link}", 0
+        break
+    else:
+        return f"**Ошибка (VK)**: Too many requests per second — {canonical_link}", 0
+
+    response_data = data.get("response", {})
+    items = response_data.get("items", [])
+    if not items:
+        return f"**Ошибка (VK)**: пост не найден — {canonical_link}", 0
+
+    post = items[0]
+    views = post.get("views", {}).get("count", 0)
+    post_owner_id = post.get("owner_id", owner_id)
+
+    title = None
+    if post_owner_id < 0:
+        group_id = -post_owner_id
+        for group in response_data.get("groups", []):
+            if group.get("id") == group_id:
+                title = group.get("name")
+                break
+    else:
+        user_id = post_owner_id
+        for profile in response_data.get("profiles", []):
+            if profile.get("id") == user_id:
+                first_name = profile.get("first_name", "")
+                last_name = profile.get("last_name", "")
+                title = (first_name + " " + last_name).strip()
+                break
+    if not title:
+        title = "VK пост"
+
+    return title, views
+
+
+def process_vk_batch(
+    batch_items: List[Tuple[str, int, str]]
+) -> dict[str, Tuple[str, int]]:
+    api_url = "https://api.vk.com/method/wall.getById"
+    posts = ",".join(f"{owner_id}_{post_id}" for _, owner_id, post_id in batch_items)
+
+    for attempt in range(1, VK_MAX_RETRIES + 1):
+        params = {
+            "posts": posts,
+            "v": VK_API_VERSION,
+            "extended": 1,
+        }
+        if VK_ACCESS_TOKEN:
+            params["access_token"] = VK_ACCESS_TOKEN
+
+        try:
+            resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
+        except Exception as exc:
+            return {
+                canonical: (f"**Ошибка (VK)**: {exc} — {canonical}", 0)
+                for canonical, _, _ in batch_items
+            }
+
+        if resp.status_code != 200:
+            return {
+                canonical: (f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical}", 0)
+                for canonical, _, _ in batch_items
+            }
+
+        data = resp.json()
+        if "error" in data:
+            error = data["error"]
+            error_msg = error.get("error_msg", "")
+            error_code = error.get("error_code")
+            if error_code == 6 or "too many requests per second" in error_msg.lower():
+                time.sleep(VK_RETRY_DELAY * attempt)
+                continue
+            return {
+                canonical: (f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical}", 0)
+                for canonical, _, _ in batch_items
+            }
+        break
+    else:
+        return {
+            canonical: (f"**Ошибка (VK)**: Too many requests per second — {canonical}", 0)
+            for canonical, _, _ in batch_items
+        }
+
+    response_data = data.get("response", {})
+    items = response_data.get("items", [])
+    groups = {group.get("id"): group for group in response_data.get("groups", [])}
+    profiles = {profile.get("id"): profile for profile in response_data.get("profiles", [])}
+    item_map = {
+        f"{item.get('owner_id')}_{item.get('id')}": item
+        for item in items
+        if item.get("owner_id") is not None and item.get("id") is not None
+    }
+
+    result: dict[str, Tuple[str, int]] = {}
+    for canonical, owner_id, post_id in batch_items:
+        key = f"{owner_id}_{post_id}"
+        post = item_map.get(key)
+        if not post:
+            result[canonical] = (f"**Ошибка (VK)**: пост не найден — {canonical}", 0)
+            continue
+
+        views = post.get("views", {}).get("count", 0)
+        post_owner_id = post.get("owner_id", owner_id)
+        title = None
+
+        if post_owner_id < 0:
+            group = groups.get(-post_owner_id)
+            if group:
+                title = group.get("name")
+        else:
+            profile = profiles.get(post_owner_id)
+            if profile:
+                first_name = profile.get("first_name", "")
+                last_name = profile.get("last_name", "")
+                title = (first_name + " " + last_name).strip()
+
+        result[canonical] = (title or "VK пост", views)
+
+    return result
+
+
+def normalize_links(links: Union[List[str], str]) -> List[str]:
+    if isinstance(links, str):
+        raw = links.splitlines()
+    else:
+        raw = []
+        for item in links:
+            raw.extend(str(item).splitlines())
+    return [line.strip() for line in raw if line.strip()]
+
+
+def _escape(text: str) -> str:
+    return html_lib.escape(text, quote=True)
+
+
+def resolve_line_item(line: Tuple[str, str, object, object]) -> Tuple[str, str, str | None, int]:
+    plat, canonical, a, b = line
+
+    if plat == "telegram":
+        username, mid = a, b
+        if not username or not mid:
+            return plat, canonical, "Telegram", 0
+        title, views = process_telegram_link(username, mid, canonical)
+        return plat, canonical, title, views
+
+    if plat == "vk":
+        owner_id, post_id = a, b
+        if owner_id is None or post_id is None:
+            return plat, canonical, "VK пост", 0
+        title, views = process_vk_link(owner_id, post_id, canonical)
+        return plat, canonical, title, views
+
+    return plat, canonical, None, 0
+
+
+def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str, int, int, List[str]]:
+    errors: List[str] = []
+
+    tg_groups = {}
+    vk_groups = {}
+
+    telegram_lines = [line for line in lines if line[0] == "telegram"]
+    vk_lines = [line for line in lines if line[0] == "vk"]
+    telegram_results = {}
+    vk_results = {}
+
+    if telegram_lines:
+        worker_count = max(1, min(MAX_WORKERS, len(telegram_lines)))
+        with ThreadPoolExecutor(max_workers=worker_count) as executor:
+            for result in executor.map(resolve_line_item, telegram_lines):
+                telegram_results[result[1]] = result
+
+    valid_vk_batch: List[Tuple[str, int, str]] = []
+    for _, canonical, owner_id, post_id in vk_lines:
+        if owner_id is not None and post_id is not None:
+            valid_vk_batch.append((canonical, owner_id, post_id))
+
+    if valid_vk_batch:
+        batch_size = max(1, VK_BATCH_SIZE)
+        for start in range(0, len(valid_vk_batch), batch_size):
+            chunk = valid_vk_batch[start:start + batch_size]
+            vk_results.update(process_vk_batch(chunk))
+
+    for plat, canonical, a, b in lines:
+        if plat == "telegram":
+            username, mid = a, b
+            if not username or not mid:
+                key = canonical
+                tg_groups.setdefault(key, {"title": "Telegram", "items": []})
+                tg_groups[key]["items"].append((canonical, 0))
+            else:
+                _, _, title, views = telegram_results.get(canonical, (plat, canonical, username, 0))
+                key = username
+                if key not in tg_groups:
+                    tg_groups[key] = {"title": title, "items": []}
+                if not tg_groups[key].get("title") or str(tg_groups[key]["title"]).startswith("**Ошибка"):
+                    tg_groups[key]["title"] = title
+                tg_groups[key]["items"].append((canonical, views))
+
+        elif plat == "vk":
+            owner_id, post_id = a, b
+            if owner_id is None or post_id is None:
+                key = canonical
+                vk_groups.setdefault(key, {"title": "VK пост", "items": []})
+                vk_groups[key]["items"].append((canonical, 0))
+            else:
+                title, views = vk_results.get(canonical, process_vk_link(owner_id, post_id, canonical))
+                key = str(owner_id)
+                if key not in vk_groups:
+                    vk_groups[key] = {"title": title, "items": []}
+                if not vk_groups[key].get("title") or str(vk_groups[key]["title"]).startswith("**Ошибка"):
+                    vk_groups[key]["title"] = title
+                vk_groups[key]["items"].append((canonical, views))
+        else:
+            errors.append(f"Неизвестная платформа: {canonical}")
+
+    tg_total_views = sum(v for g in tg_groups.values() for _, v in g["items"])
+    vk_total_views = sum(v for g in vk_groups.values() for _, v in g["items"])
+
+    tg_sorted = sorted(
+        tg_groups.items(),
+        key=lambda kv: sum(v for _, v in kv[1]["items"]),
+        reverse=True,
+    )
+    vk_sorted = sorted(
+        vk_groups.items(),
+        key=lambda kv: sum(v for _, v in kv[1]["items"]),
+        reverse=True,
+    )
+
+    html_lines: List[str] = []
+    text_lines: List[str] = []
+
+    # --- Telegram ---
+    html_lines.append("<h2>Telegram</h2>")
+    html_lines.append(f'Суммарно посты собрали <b>{human_format_views(tg_total_views)}</b> просмотров.')
+    text_lines.append("Telegram")
+    text_lines.append(f"Суммарно посты собрали {human_format_views(tg_total_views)} просмотров.")
+    if tg_sorted:
+        html_lines.append("<ol>")
+        for idx, (_, data) in enumerate(tg_sorted, start=1):
+            title = data["title"] or "Telegram"
+            items = data["items"]
+            first_link, _first_views = items[0]
+
+            title_html = _escape(title)
+            first_link_html = _escape(first_link)
+            line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
+            if len(items) > 1:
+                for link2, _v in items[1:]:
+                    line_html += f' + <a href="{_escape(link2)}">ещё</a>'
+            views_str = " + ".join(human_format_views(v) for _, v in items)
+            line_html += f" — {views_str}</li>"
+            html_lines.append(line_html)
+
+            line_text = f"{idx}. {title} ({first_link})"
+            if len(items) > 1:
+                extra_links = ", ".join(link2 for link2, _v in items[1:])
+                line_text += f" + ещё: {extra_links}"
+            line_text += f" — {views_str}"
+            text_lines.append(line_text)
+        html_lines.append("</ol>")
+    else:
+        html_lines.append("<i>Нет ссылок на Telegram</i>")
+        text_lines.append("Нет ссылок на Telegram")
+    html_lines.append("<br/>")
+    text_lines.append("")
+
+    # --- VK ---
+    html_lines.append("<h2>ВКонтакте</h2>")
+    html_lines.append(f'Суммарно посты собрали <b>{human_format_views(vk_total_views)}</b> просмотров.')
+    text_lines.append("ВКонтакте")
+    text_lines.append(f"Суммарно посты собрали {human_format_views(vk_total_views)} просмотров.")
+    if vk_sorted:
+        html_lines.append("<ol>")
+        for idx, (_, data) in enumerate(vk_sorted, start=1):
+            title = data["title"] or "VK пост"
+            items = data["items"]
+            first_link, _first_views = items[0]
+
+            title_html = _escape(title)
+            first_link_html = _escape(first_link)
+            line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
+            if len(items) > 1:
+                for link2, _v in items[1:]:
+                    line_html += f' + <a href="{_escape(link2)}">ещё</a>'
+            views_str = " + ".join(human_format_views(v) for _, v in items)
+            line_html += f" — {views_str}</li>"
+            html_lines.append(line_html)
+
+            line_text = f"{idx}. {title} ({first_link})"
+            if len(items) > 1:
+                extra_links = ", ".join(link2 for link2, _v in items[1:])
+                line_text += f" + ещё: {extra_links}"
+            line_text += f" — {views_str}"
+            text_lines.append(line_text)
+        html_lines.append("</ol>")
+    else:
+        html_lines.append("<i>Нет ссылок на ВКонтакте</i>")
+        text_lines.append("Нет ссылок на ВКонтакте")
+    html_lines.append("<br/>")
+    text_lines.append("")
+
+    if errors:
+        html_lines.append("<h2>Ошибки</h2>")
+        html_lines.append("<ul>")
+        for err in errors:
+            html_lines.append(f"<li>{_escape(err)}</li>")
+            text_lines.append(f"- {err}")
+        html_lines.append("</ul>")
+        html_lines.append("<br/>")
+        text_lines.append("")
+
+    return "\n".join(html_lines), "\n".join(text_lines), tg_total_views, vk_total_views, errors
+
+
+@app.get("/health")
+def health_check():
+    return {"status": "ok"}
+
+
+@app.post("/parse", response_model=ParseResponse)
+def parse_links(payload: ParseRequest):
+    raw_lines = normalize_links(payload.links)
+    if not raw_lines:
+        return ParseResponse(html="", text="", telegram_total=0, vk_total=0, errors=["Нет ссылок для обработки."])
+
+    seen = set()
+    lines: List[Tuple[str, str, object, object]] = []
+    for link in raw_lines:
+        plat = detect_platform(link)
+        if plat == "telegram":
+            canonical, username, mid = canonicalize_tg_link(link)
+            if not canonical:
+                canonical = link
+                username = None
+                mid = None
+            if canonical in seen:
+                continue
+            seen.add(canonical)
+            lines.append(("telegram", canonical, username, mid))
+        elif plat == "vk":
+            canonical, owner_id, post_id = canonicalize_vk_link(link)
+            if not canonical:
+                canonical = link
+            if canonical in seen:
+                continue
+            seen.add(canonical)
+            lines.append(("vk", canonical, owner_id, post_id))
+        else:
+            lines.append(("unknown", link, None, None))
+
+    html, text, tg_total, vk_total, errors = build_output(lines)
+    return ParseResponse(
+        html=html,
+        text=text,
+        telegram_total=tg_total,
+        vk_total=vk_total,
+        errors=errors,
+    )