Dink0 commited on
Commit
62becde
·
verified ·
1 Parent(s): 081d8c6

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -931
app.py DELETED
@@ -1,931 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import asyncio
4
- from threading import Lock
5
- import time
6
- import os
7
- import re
8
- import html as html_lib
9
- from typing import List, Tuple, Union
10
-
11
- import requests
12
- from fastapi import FastAPI
13
- from fastapi.middleware.cors import CORSMiddleware
14
- from pydantic import BaseModel
15
-
16
- try:
17
- from telethon import TelegramClient
18
- from telethon.sessions import StringSession
19
- except Exception:
20
- TelegramClient = None
21
- StringSession = None
22
-
23
- try:
24
- from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
25
- except Exception:
26
- PlaywrightTimeoutError = Exception
27
- sync_playwright = None
28
-
29
- APP_NAME = "pr-tool-backend"
30
-
31
- VK_API_VERSION = os.getenv("VK_API_VERSION", "5.131")
32
- VK_ACCESS_TOKEN = os.getenv("VK_ACCESS_TOKEN", "")
33
- TELEGRAM_API_ID = int(os.getenv("TELEGRAM_API_ID", "0") or "0")
34
- TELEGRAM_API_HASH = os.getenv("TELEGRAM_API_HASH", "")
35
- TELEGRAM_STRING_SESSION = os.getenv("TELEGRAM_STRING_SESSION", "")
36
- REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "15"))
37
- VK_MAX_RETRIES = int(os.getenv("VK_MAX_RETRIES", "5"))
38
- VK_RETRY_DELAY = float(os.getenv("VK_RETRY_DELAY", "0.45"))
39
- VK_BATCH_SIZE = int(os.getenv("VK_BATCH_SIZE", "100"))
40
- PLAYWRIGHT_GOTO_TIMEOUT_MS = int(os.getenv("PLAYWRIGHT_GOTO_TIMEOUT_MS", "20000"))
41
- ENABLE_TELEGRAM_BROWSER_FALLBACK = os.getenv("ENABLE_TELEGRAM_BROWSER_FALLBACK", "1") != "0"
42
-
43
- _TELEGRAM_BROWSER_LOCK = Lock()
44
-
45
- app = FastAPI(title=APP_NAME)
46
-
47
- # Для простоты разрешаем все источники. Можно сузить список доменов в проде.
48
- app.add_middleware(
49
- CORSMiddleware,
50
- allow_origins=["*"],
51
- allow_credentials=False,
52
- allow_methods=["POST", "GET", "OPTIONS"],
53
- allow_headers=["*"]
54
- )
55
-
56
-
57
- class ParseRequest(BaseModel):
58
- links: Union[List[str], str]
59
-
60
-
61
- class ParseResponse(BaseModel):
62
- html: str
63
- text: str
64
- telegram_total: int
65
- vk_total: int
66
- errors: List[str]
67
-
68
-
69
- # ---------- Вспомогательные функции ----------
70
-
71
- def clean_telegram_title(raw: str, fallback: str) -> str:
72
- title = re.sub(
73
- r"<i\b[^>]*class=[\"'][^\"']*emoji[^\"']*[\"'][^>]*>.*?</i>",
74
- "",
75
- (raw or "").strip(),
76
- flags=re.IGNORECASE | re.DOTALL,
77
- )
78
- title = re.sub(r"<[^>]+>", "", title)
79
- title = html_lib.unescape(title)
80
- title = re.sub(r"\s*[-|]\s*Telegram\s*$", "", title, flags=re.IGNORECASE)
81
- title = re.sub(
82
- r"[\U0001F1E6-\U0001F1FF\U0001F300-\U0001FAFF\u2600-\u27BF\uFE0E\uFE0F]",
83
- "",
84
- title,
85
- )
86
- title = re.sub(r"\s+", " ", title).strip()
87
- return title or fallback
88
-
89
- def human_format_views(num: int) -> str:
90
- if num >= 1_000_000:
91
- value = num / 1_000_000
92
- suffix = "M"
93
- else:
94
- value = num / 1000
95
- suffix = "K"
96
-
97
- rounded = round(value, 1)
98
- if rounded.is_integer():
99
- formatted = f"{int(rounded)}{suffix}"
100
- else:
101
- formatted = f"{rounded:.1f}{suffix}"
102
-
103
- return formatted.replace(".", ",")
104
-
105
-
106
- def detect_platform(link: str) -> str:
107
- link = link.strip().lower()
108
- if "t.me/" in link or "telegram.me/" in link:
109
- return "telegram"
110
- if "vk.com/wall" in link or "vk.ru/wall" in link:
111
- return "vk"
112
- return ""
113
-
114
-
115
- def canonicalize_tg_link(link: str) -> Tuple[str | None, str | None, str | None]:
116
- link = link.strip()
117
- m = re.match(r"https?://(?:t(?:elegram)?\.me)/(?:s/)?([^/]+)/(?P<id>\d+)", link, re.IGNORECASE)
118
- if not m:
119
- return None, None, None
120
- username = m.group(1)
121
- message_id = m.group("id")
122
- canonical = f"https://t.me/{username}/{message_id}"
123
- return canonical, username, message_id
124
-
125
-
126
- def canonicalize_vk_link(link: str) -> Tuple[str | None, int | None, str | None]:
127
- link = link.strip()
128
- m = re.search(r"(?:vk\.com|vk\.ru)/wall(-?\d+)_(\d+)", link)
129
- if not m:
130
- return None, None, None
131
- owner_id = int(m.group(1))
132
- post_id = m.group(2)
133
- canonical = f"https://vk.com/wall{owner_id}_{post_id}"
134
- return canonical, owner_id, post_id
135
-
136
-
137
- def _telethon_ready() -> bool:
138
- return bool(
139
- TelegramClient
140
- and StringSession
141
- and TELEGRAM_API_ID
142
- and TELEGRAM_API_HASH
143
- and TELEGRAM_STRING_SESSION
144
- )
145
-
146
-
147
- def process_telegram_batch(batch_items: List[Tuple[str, str, str]]) -> dict[str, Tuple[str, int]]:
148
- if not batch_items:
149
- return {}
150
-
151
- if not _telethon_ready():
152
- return {
153
- canonical: process_telegram_link(username, message_id, canonical)
154
- for canonical, username, message_id in batch_items
155
- }
156
-
157
- async def _run_batch() -> dict[str, Tuple[str, int]]:
158
- results: dict[str, Tuple[str, int]] = {}
159
- grouped: dict[str, List[Tuple[str, int]]] = {}
160
-
161
- for canonical, username, message_id in batch_items:
162
- grouped.setdefault(username, []).append((canonical, int(message_id)))
163
-
164
- client = TelegramClient(
165
- StringSession(TELEGRAM_STRING_SESSION),
166
- TELEGRAM_API_ID,
167
- TELEGRAM_API_HASH,
168
- request_retries=2,
169
- connection_retries=2,
170
- timeout=REQUEST_TIMEOUT,
171
- )
172
-
173
- async with client:
174
- for username, items in grouped.items():
175
- try:
176
- entity = await client.get_entity(username)
177
- except Exception as exc:
178
- for canonical, _ in items:
179
- results[canonical] = (f"Ошибка (TG) при обработке {canonical}: {exc}", 0)
180
- continue
181
-
182
- entity_title = getattr(entity, "title", None)
183
- if not entity_title:
184
- first_name = getattr(entity, "first_name", "")
185
- last_name = getattr(entity, "last_name", "")
186
- entity_title = (f"{first_name} {last_name}").strip()
187
- title = clean_telegram_title(entity_title or "", username)
188
-
189
- ids = [message_id for _, message_id in items]
190
- try:
191
- messages = await client.get_messages(entity, ids=ids)
192
- except Exception as exc:
193
- for canonical, _ in items:
194
- results[canonical] = (f"Ошибка (TG) при обработке {canonical}: {exc}", 0)
195
- continue
196
-
197
- if not isinstance(messages, list):
198
- messages = [messages]
199
-
200
- message_map = {}
201
- for message in messages:
202
- if message is None:
203
- continue
204
- message_id = getattr(message, "id", None)
205
- if message_id is None:
206
- continue
207
- message_map[int(message_id)] = message
208
-
209
- for canonical, message_id in items:
210
- message = message_map.get(message_id)
211
- if message is None:
212
- results[canonical] = (f"Ошибка (TG): пост не найден — {canonical}", 0)
213
- continue
214
- views = int(getattr(message, "views", 0) or 0)
215
- results[canonical] = (title, views)
216
-
217
- return results
218
-
219
- try:
220
- return asyncio.run(_run_batch())
221
- except Exception:
222
- return {
223
- canonical: process_telegram_link(username, message_id, canonical)
224
- for canonical, username, message_id in batch_items
225
- }
226
-
227
-
228
- def process_telegram_link(channel_username: str, message_id: str, canonical_link: str) -> Tuple[str, int]:
229
- # Набор альтернативных адресов (если t.me не резолвится)
230
- urls = [
231
- f"https://t.me/{channel_username}/{message_id}?embed=1",
232
- f"https://t.me/{channel_username}/{message_id}?embed=1&single=1",
233
- f"https://t.me/{channel_username}/{message_id}?embed=1&mode=tme",
234
- f"https://t.me/s/{channel_username}/{message_id}",
235
- f"https://t.me/{channel_username}/{message_id}",
236
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}?embed=1",
237
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}?embed=1&single=1",
238
- f"https://r.jina.ai/http://t.me/s/{channel_username}/{message_id}",
239
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}",
240
- ]
241
- headers = {"User-Agent": "Mozilla/5.0"}
242
- post_headers = {
243
- "User-Agent": "Mozilla/5.0",
244
- "Content-Type": "application/x-www-form-urlencoded",
245
- }
246
-
247
- def _parse_views_number(s: str) -> int:
248
- if not s:
249
- return 0
250
- s = s.strip().replace("\u00a0", "").replace(" ", "").replace(",", ".")
251
- m2 = re.match(r"([\d\.]+)\s*([kKmM]?)", s)
252
- if not m2:
253
- return 0
254
- val = float(m2.group(1))
255
- suf = m2.group(2).lower()
256
- if suf == "k":
257
- val *= 1_000
258
- elif suf == "m":
259
- val *= 1_000_000
260
- return int(val)
261
-
262
- def _fetch_ok_responses(url_list):
263
- texts: List[str] = []
264
- last_err = None
265
- for u in url_list:
266
- try:
267
- resp = requests.get(u, headers=headers, timeout=REQUEST_TIMEOUT)
268
- if resp.status_code == 200 and resp.text:
269
- texts.append(resp.text)
270
-
271
- # Встроенный tg-виджет сам делает POST на тот же URL с _rl=1.
272
- # Иногда именно этот ответ содержит уже дорисованный widget DOM с просмотрами.
273
- if "tgme_widget_message_views" not in resp.text:
274
- post_resp = requests.post(
275
- u,
276
- headers=post_headers,
277
- data="_rl=1",
278
- timeout=REQUEST_TIMEOUT,
279
- )
280
- if post_resp.status_code == 200 and post_resp.text:
281
- texts.append(post_resp.text)
282
- except Exception as exc:
283
- last_err = exc
284
- if texts:
285
- return texts
286
- raise last_err or Exception("Не удалось получить страницу Telegram")
287
-
288
- def _strip_emoji(s: str) -> str:
289
- # Убираем emoji/pictographs и variation selectors.
290
- s = re.sub(
291
- r"[\U0001F1E6-\U0001F1FF\U0001F300-\U0001FAFF\u2600-\u27BF\uFE0E\uFE0F]",
292
- "",
293
- s,
294
- )
295
- return re.sub(r"\s+", " ", s).strip()
296
-
297
- def _clean_title(raw: str, fallback: str) -> str:
298
- title = re.sub(
299
- r"<i\b[^>]*class=[\"'][^\"']*emoji[^\"']*[\"'][^>]*>.*?</i>",
300
- "",
301
- (raw or "").strip(),
302
- flags=re.IGNORECASE | re.DOTALL,
303
- )
304
- title = re.sub(r"<[^>]+>", "", title)
305
- title = html_lib.unescape(title)
306
- title = re.sub(r"\s*[-|]\s*Telegram\s*$", "", title, flags=re.IGNORECASE)
307
- title = _strip_emoji(title)
308
- return title or fallback
309
-
310
- def _is_placeholder_title(raw: str | None) -> bool:
311
- if not raw:
312
- return True
313
- normalized = _clean_title(raw, "").strip().lower()
314
- placeholders = {
315
- "",
316
- "telegram",
317
- "telegram widget",
318
- "telegram - a new era of messaging",
319
- "telegram – a new era of messaging",
320
- }
321
- if normalized in placeholders:
322
- return True
323
- if normalized.startswith("telegram:"):
324
- return True
325
- if "a new era of messaging" in normalized:
326
- return True
327
- return False
328
-
329
- def _is_username_like_title(raw: str | None) -> bool:
330
- if not raw:
331
- return False
332
- normalized = _clean_title(raw, "").strip().lower()
333
- username_variants = {
334
- channel_username.lower(),
335
- f"@{channel_username.lower()}",
336
- f"t.me/{channel_username.lower()}",
337
- }
338
- return normalized in username_variants
339
-
340
- def _extract_views(html: str) -> int:
341
- patterns = [
342
- # Виджет Telegram
343
- r'class="tgme_widget_message_views[^"]*">([\d\s\.,kKmM]+)<',
344
- # Резервный вариант из meta-description, но только если рядом явно "views"
345
- r'content="[^"]*?([\d\s\.,kKmM]+)\s+views[^"]*"',
346
- r'content="[^"]*?(?:Просмотры|просмотры|просмотров)[:\s]*([\d\s\.,kKmM]+)[^"]*"',
347
- # Текстовый fallback (например, через jina)
348
- r'(?:^|\n)\s*(?:Views|views|Просмотры|просмотры|Просмотров|просмотров)\s*[:\-]?\s*([\d\s\.,kKmM]+)\b',
349
- r'([\d\s\.,kKmM]+)\s*(?:views|Views|просмотров|Просмотров)\b',
350
- r'(?:👁|👀)\s*([\d\s\.,kKmM]+)\b',
351
- ]
352
- for pattern in patterns:
353
- m = re.search(pattern, html, re.IGNORECASE)
354
- if m:
355
- parsed = _parse_views_number(m.group(1))
356
- if parsed > 0:
357
- return parsed
358
- return 0
359
-
360
- def _extract_text_title(raw_text: str) -> str | None:
361
- looks_like_html = bool(re.search(r"<(?:!DOCTYPE|html|head|body|meta|script|div)\b", raw_text, re.IGNORECASE))
362
- patterns = [
363
- r'(?:^|\n)\s*Title:\s*(.+)',
364
- r'(?:^|\n)\s*Channel:\s*(.+)',
365
- r'(?:^|\n)\s*#\s+(.+)',
366
- ]
367
- for pattern in patterns:
368
- m = re.search(pattern, raw_text, re.IGNORECASE)
369
- if m:
370
- return m.group(1).strip()
371
-
372
- if looks_like_html:
373
- return None
374
-
375
- for line in raw_text.splitlines():
376
- line = line.strip()
377
- if not line:
378
- continue
379
- if line.startswith("http://") or line.startswith("https://"):
380
- continue
381
- if "views" in line.lower() or "просмотр" in line.lower():
382
- continue
383
- if _is_username_like_title(line):
384
- continue
385
- if len(line) <= 120:
386
- return line
387
- return None
388
-
389
- def _extract_title(raw_text: str) -> str | None:
390
- candidates: List[str] = []
391
-
392
- m_title_meta = re.search(
393
- r"<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
394
- raw_text,
395
- re.IGNORECASE,
396
- )
397
- if m_title_meta:
398
- candidates.append(m_title_meta.group(1).strip())
399
-
400
- m_twitter_title = re.search(
401
- r"<meta[^>]*property=[\"']twitter:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
402
- raw_text,
403
- re.IGNORECASE,
404
- )
405
- if m_twitter_title:
406
- candidates.append(m_twitter_title.group(1).strip())
407
-
408
- m_site_name = re.search(
409
- r"<meta[^>]*property=[\"']og:site_name[\"'][^>]*content=[\"']([^\"']+)[\"']",
410
- raw_text,
411
- re.IGNORECASE,
412
- )
413
- if m_site_name:
414
- candidates.append(m_site_name.group(1).strip())
415
-
416
- m_author = re.search(
417
- r'class="tgme_widget_message_author_name".*?<span[^>]*>(.*?)</span>',
418
- raw_text,
419
- re.IGNORECASE | re.DOTALL,
420
- )
421
- if m_author:
422
- candidates.append(m_author.group(1).strip())
423
-
424
- m_owner = re.search(
425
- r'class="tgme_widget_message_owner_name".*?<span[^>]*>(.*?)</span>',
426
- raw_text,
427
- re.IGNORECASE | re.DOTALL,
428
- )
429
- if m_owner:
430
- candidates.append(m_owner.group(1).strip())
431
-
432
- text_title = _extract_text_title(raw_text)
433
- if text_title:
434
- candidates.append(text_title)
435
-
436
- for candidate in candidates:
437
- if not _is_placeholder_title(candidate):
438
- return candidate
439
- return None
440
-
441
- def _pick_widget_html(raw_text: str) -> str | None:
442
- candidates = [raw_text]
443
- iframe_matches = re.findall(r"<iframe\b[^>]*>(.*?)</iframe>", raw_text, re.IGNORECASE | re.DOTALL)
444
- candidates.extend(iframe_matches)
445
-
446
- for candidate in candidates:
447
- if (
448
- "tgme_widget_message_views" in candidate
449
- or "tgme_widget_message_owner_name" in candidate
450
- or "tgme_widget_message_author_name" in candidate
451
- ):
452
- return candidate
453
- return None
454
-
455
- def _fetch_browser_response() -> str | None:
456
- if not ENABLE_TELEGRAM_BROWSER_FALLBACK or sync_playwright is None:
457
- return None
458
-
459
- browser_urls = [
460
- f"https://t.me/{channel_username}/{message_id}?embed=1&single=1",
461
- f"https://t.me/{channel_username}/{message_id}?embed=1",
462
- f"https://t.me/{channel_username}/{message_id}",
463
- ]
464
-
465
- with _TELEGRAM_BROWSER_LOCK:
466
- try:
467
- with sync_playwright() as playwright:
468
- browser = playwright.chromium.launch(
469
- headless=True,
470
- args=["--no-sandbox", "--disable-dev-shm-usage"],
471
- )
472
- context = browser.new_context(
473
- user_agent=headers["User-Agent"],
474
- viewport={"width": 1280, "height": 900},
475
- )
476
- page = context.new_page()
477
-
478
- try:
479
- for url in browser_urls:
480
- try:
481
- page.goto(url, wait_until="domcontentloaded", timeout=PLAYWRIGHT_GOTO_TIMEOUT_MS)
482
- except PlaywrightTimeoutError:
483
- pass
484
- except Exception:
485
- continue
486
-
487
- try:
488
- page.wait_for_timeout(700)
489
- page.wait_for_selector(
490
- ".tgme_widget_message_views, .tgme_widget_message_owner_name, .tgme_widget_message_author_name",
491
- timeout=4000,
492
- )
493
- except PlaywrightTimeoutError:
494
- pass
495
- except Exception:
496
- pass
497
-
498
- frame_htmls: List[str] = []
499
- try:
500
- frame_htmls.append(page.content())
501
- except Exception:
502
- pass
503
-
504
- for frame in page.frames:
505
- if frame == page.main_frame:
506
- continue
507
- try:
508
- frame_htmls.append(frame.content())
509
- except Exception:
510
- continue
511
-
512
- for frame_html in frame_htmls:
513
- widget_html = _pick_widget_html(frame_html)
514
- if widget_html:
515
- return widget_html
516
- finally:
517
- context.close()
518
- browser.close()
519
- except Exception:
520
- return None
521
-
522
- return None
523
-
524
- try:
525
- responses = _fetch_ok_responses(urls)
526
-
527
- title_raw = None
528
- views = 0
529
- for response_text in responses:
530
- candidate_title = _extract_title(response_text)
531
- if candidate_title:
532
- if not title_raw:
533
- title_raw = candidate_title
534
- elif _is_username_like_title(title_raw) and not _is_username_like_title(candidate_title):
535
- title_raw = candidate_title
536
-
537
- candidate_views = _extract_views(response_text)
538
- if candidate_views > views:
539
- views = candidate_views
540
-
541
- if title_raw and not _is_username_like_title(title_raw) and views > 0:
542
- break
543
-
544
- if not title_raw or _is_username_like_title(title_raw) or views <= 0:
545
- browser_response = _fetch_browser_response()
546
- if browser_response:
547
- browser_title = _extract_title(browser_response)
548
- browser_views = _extract_views(browser_response)
549
- if browser_title and (not title_raw or _is_username_like_title(title_raw)):
550
- title_raw = browser_title
551
- if browser_views > views:
552
- views = browser_views
553
-
554
- title = _clean_title(title_raw or "", channel_username)
555
-
556
- return title, views
557
-
558
- except Exception as exc:
559
- return f"Ошибка (TG) при обработке {canonical_link}: {exc}", 0
560
-
561
-
562
- def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[str, int]:
563
- posts_param = f"{owner_id}_{post_id}"
564
- api_url = "https://api.vk.com/method/wall.getById"
565
-
566
- for attempt in range(1, VK_MAX_RETRIES + 1):
567
- params = {
568
- "posts": posts_param,
569
- "v": VK_API_VERSION,
570
- "extended": 1,
571
- }
572
- if VK_ACCESS_TOKEN:
573
- params["access_token"] = VK_ACCESS_TOKEN
574
-
575
- try:
576
- resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
577
- except Exception as exc:
578
- return f"**Ошибка (VK)**: {exc} — {canonical_link}", 0
579
-
580
- if resp.status_code != 200:
581
- return f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical_link}", 0
582
-
583
- data = resp.json()
584
- if "error" in data:
585
- error = data["error"]
586
- error_msg = error.get("error_msg", "")
587
- error_code = error.get("error_code")
588
- if error_code == 6 or "too many requests per second" in error_msg.lower():
589
- time.sleep(VK_RETRY_DELAY * attempt)
590
- continue
591
- return f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical_link}", 0
592
- break
593
- else:
594
- return f"**Ошибка (VK)**: Too many requests per second — {canonical_link}", 0
595
-
596
- response_data = data.get("response", {})
597
- items = response_data.get("items", [])
598
- if not items:
599
- return f"**Ошибка (VK)**: пост не найден — {canonical_link}", 0
600
-
601
- post = items[0]
602
- views = post.get("views", {}).get("count", 0)
603
- post_owner_id = post.get("owner_id", owner_id)
604
-
605
- title = None
606
- if post_owner_id < 0:
607
- group_id = -post_owner_id
608
- for group in response_data.get("groups", []):
609
- if group.get("id") == group_id:
610
- title = group.get("name")
611
- break
612
- else:
613
- user_id = post_owner_id
614
- for profile in response_data.get("profiles", []):
615
- if profile.get("id") == user_id:
616
- first_name = profile.get("first_name", "")
617
- last_name = profile.get("last_name", "")
618
- title = (first_name + " " + last_name).strip()
619
- break
620
- if not title:
621
- title = "VK пост"
622
-
623
- return title, views
624
-
625
-
626
- def process_vk_batch(
627
- batch_items: List[Tuple[str, int, str]]
628
- ) -> dict[str, Tuple[str, int]]:
629
- api_url = "https://api.vk.com/method/wall.getById"
630
- posts = ",".join(f"{owner_id}_{post_id}" for _, owner_id, post_id in batch_items)
631
-
632
- for attempt in range(1, VK_MAX_RETRIES + 1):
633
- params = {
634
- "posts": posts,
635
- "v": VK_API_VERSION,
636
- "extended": 1,
637
- }
638
- if VK_ACCESS_TOKEN:
639
- params["access_token"] = VK_ACCESS_TOKEN
640
-
641
- try:
642
- resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
643
- except Exception as exc:
644
- return {
645
- canonical: (f"**Ошибка (VK)**: {exc} — {canonical}", 0)
646
- for canonical, _, _ in batch_items
647
- }
648
-
649
- if resp.status_code != 200:
650
- return {
651
- canonical: (f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical}", 0)
652
- for canonical, _, _ in batch_items
653
- }
654
-
655
- data = resp.json()
656
- if "error" in data:
657
- error = data["error"]
658
- error_msg = error.get("error_msg", "")
659
- error_code = error.get("error_code")
660
- if error_code == 6 or "too many requests per second" in error_msg.lower():
661
- time.sleep(VK_RETRY_DELAY * attempt)
662
- continue
663
- return {
664
- canonical: (f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical}", 0)
665
- for canonical, _, _ in batch_items
666
- }
667
- break
668
- else:
669
- return {
670
- canonical: (f"**Ошибка (VK)**: Too many requests per second — {canonical}", 0)
671
- for canonical, _, _ in batch_items
672
- }
673
-
674
- response_data = data.get("response", {})
675
- items = response_data.get("items", [])
676
- groups = {group.get("id"): group for group in response_data.get("groups", [])}
677
- profiles = {profile.get("id"): profile for profile in response_data.get("profiles", [])}
678
- item_map = {
679
- f"{item.get('owner_id')}_{item.get('id')}": item
680
- for item in items
681
- if item.get("owner_id") is not None and item.get("id") is not None
682
- }
683
-
684
- result: dict[str, Tuple[str, int]] = {}
685
- for canonical, owner_id, post_id in batch_items:
686
- key = f"{owner_id}_{post_id}"
687
- post = item_map.get(key)
688
- if not post:
689
- result[canonical] = (f"**Ошибка (VK)**: пост не найден — {canonical}", 0)
690
- continue
691
-
692
- views = post.get("views", {}).get("count", 0)
693
- post_owner_id = post.get("owner_id", owner_id)
694
- title = None
695
-
696
- if post_owner_id < 0:
697
- group = groups.get(-post_owner_id)
698
- if group:
699
- title = group.get("name")
700
- else:
701
- profile = profiles.get(post_owner_id)
702
- if profile:
703
- first_name = profile.get("first_name", "")
704
- last_name = profile.get("last_name", "")
705
- title = (first_name + " " + last_name).strip()
706
-
707
- result[canonical] = (title or "VK пост", views)
708
-
709
- return result
710
-
711
-
712
- def normalize_links(links: Union[List[str], str]) -> List[str]:
713
- if isinstance(links, str):
714
- raw = links.splitlines()
715
- else:
716
- raw = []
717
- for item in links:
718
- raw.extend(str(item).splitlines())
719
- return [line.strip() for line in raw if line.strip()]
720
-
721
-
722
- def _escape(text: str) -> str:
723
- return html_lib.escape(text, quote=True)
724
-
725
- def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str, int, int, List[str]]:
726
- errors: List[str] = []
727
-
728
- tg_groups = {}
729
- vk_groups = {}
730
-
731
- telegram_lines = [line for line in lines if line[0] == "telegram"]
732
- vk_lines = [line for line in lines if line[0] == "vk"]
733
- telegram_results = {}
734
- vk_results = {}
735
-
736
- if telegram_lines:
737
- valid_telegram_batch: List[Tuple[str, str, str]] = []
738
- for _, canonical, username, mid in telegram_lines:
739
- if username and mid:
740
- valid_telegram_batch.append((canonical, username, mid))
741
- if valid_telegram_batch:
742
- telegram_results = process_telegram_batch(valid_telegram_batch)
743
-
744
- valid_vk_batch: List[Tuple[str, int, str]] = []
745
- for _, canonical, owner_id, post_id in vk_lines:
746
- if owner_id is not None and post_id is not None:
747
- valid_vk_batch.append((canonical, owner_id, post_id))
748
-
749
- if valid_vk_batch:
750
- batch_size = max(1, VK_BATCH_SIZE)
751
- for start in range(0, len(valid_vk_batch), batch_size):
752
- chunk = valid_vk_batch[start:start + batch_size]
753
- vk_results.update(process_vk_batch(chunk))
754
-
755
- for plat, canonical, a, b in lines:
756
- if plat == "telegram":
757
- username, mid = a, b
758
- if not username or not mid:
759
- key = canonical
760
- tg_groups.setdefault(key, {"title": "Telegram", "items": []})
761
- tg_groups[key]["items"].append((canonical, 0))
762
- else:
763
- title, views = telegram_results.get(canonical, (username, 0))
764
- key = username
765
- if key not in tg_groups:
766
- tg_groups[key] = {"title": title, "items": []}
767
- if not tg_groups[key].get("title") or str(tg_groups[key]["title"]).startswith("**Ошибка"):
768
- tg_groups[key]["title"] = title
769
- tg_groups[key]["items"].append((canonical, views))
770
-
771
- elif plat == "vk":
772
- owner_id, post_id = a, b
773
- if owner_id is None or post_id is None:
774
- key = canonical
775
- vk_groups.setdefault(key, {"title": "VK пост", "items": []})
776
- vk_groups[key]["items"].append((canonical, 0))
777
- else:
778
- title, views = vk_results.get(canonical, process_vk_link(owner_id, post_id, canonical))
779
- key = str(owner_id)
780
- if key not in vk_groups:
781
- vk_groups[key] = {"title": title, "items": []}
782
- if not vk_groups[key].get("title") or str(vk_groups[key]["title"]).startswith("**Ошибка"):
783
- vk_groups[key]["title"] = title
784
- vk_groups[key]["items"].append((canonical, views))
785
- else:
786
- errors.append(f"Неизвестная платформа: {canonical}")
787
-
788
- tg_total_views = sum(v for g in tg_groups.values() for _, v in g["items"])
789
- vk_total_views = sum(v for g in vk_groups.values() for _, v in g["items"])
790
-
791
- tg_sorted = sorted(
792
- tg_groups.items(),
793
- key=lambda kv: sum(v for _, v in kv[1]["items"]),
794
- reverse=True,
795
- )
796
- vk_sorted = sorted(
797
- vk_groups.items(),
798
- key=lambda kv: sum(v for _, v in kv[1]["items"]),
799
- reverse=True,
800
- )
801
-
802
- html_lines: List[str] = []
803
- text_lines: List[str] = []
804
-
805
- # --- Telegram ---
806
- html_lines.append("<h2>Telegram</h2>")
807
- html_lines.append(f'Суммарно посты собрали <b>{human_format_views(tg_total_views)}</b> просмотров.')
808
- text_lines.append("Telegram")
809
- text_lines.append(f"Суммарно посты собрали {human_format_views(tg_total_views)} просмотров.")
810
- if tg_sorted:
811
- html_lines.append("<ol>")
812
- for idx, (_, data) in enumerate(tg_sorted, start=1):
813
- title = data["title"] or "Telegram"
814
- items = data["items"]
815
- first_link, _first_views = items[0]
816
-
817
- title_html = _escape(title)
818
- first_link_html = _escape(first_link)
819
- line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
820
- if len(items) > 1:
821
- for link2, _v in items[1:]:
822
- line_html += f' + <a href="{_escape(link2)}">ещё</a>'
823
- views_str = " + ".join(human_format_views(v) for _, v in items)
824
- line_html += f" — {views_str}</li>"
825
- html_lines.append(line_html)
826
-
827
- line_text = f"{idx}. {title} ({first_link})"
828
- if len(items) > 1:
829
- extra_links = ", ".join(link2 for link2, _v in items[1:])
830
- line_text += f" + ещё: {extra_links}"
831
- line_text += f" — {views_str}"
832
- text_lines.append(line_text)
833
- html_lines.append("</ol>")
834
- else:
835
- html_lines.append("<i>Нет ссылок на Telegram</i>")
836
- text_lines.append("Нет ссылок на Telegram")
837
- html_lines.append("<br/>")
838
- text_lines.append("")
839
-
840
- # --- VK ---
841
- html_lines.append("<h2>ВКонтакте</h2>")
842
- html_lines.append(f'Суммарно посты собрали <b>{human_format_views(vk_total_views)}</b> просмотров.')
843
- text_lines.append("ВКонтакте")
844
- text_lines.append(f"Суммарно посты собрали {human_format_views(vk_total_views)} просмотров.")
845
- if vk_sorted:
846
- html_lines.append("<ol>")
847
- for idx, (_, data) in enumerate(vk_sorted, start=1):
848
- title = data["title"] or "VK пост"
849
- items = data["items"]
850
- first_link, _first_views = items[0]
851
-
852
- title_html = _escape(title)
853
- first_link_html = _escape(first_link)
854
- line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
855
- if len(items) > 1:
856
- for link2, _v in items[1:]:
857
- line_html += f' + <a href="{_escape(link2)}">ещё</a>'
858
- views_str = " + ".join(human_format_views(v) for _, v in items)
859
- line_html += f" — {views_str}</li>"
860
- html_lines.append(line_html)
861
-
862
- line_text = f"{idx}. {title} ({first_link})"
863
- if len(items) > 1:
864
- extra_links = ", ".join(link2 for link2, _v in items[1:])
865
- line_text += f" + ещё: {extra_links}"
866
- line_text += f" — {views_str}"
867
- text_lines.append(line_text)
868
- html_lines.append("</ol>")
869
- else:
870
- html_lines.append("<i>Нет ссылок на ВКонтакте</i>")
871
- text_lines.append("Нет ссылок на ВКонтакте")
872
- html_lines.append("<br/>")
873
- text_lines.append("")
874
-
875
- if errors:
876
- html_lines.append("<h2>Ошибки</h2>")
877
- html_lines.append("<ul>")
878
- for err in errors:
879
- html_lines.append(f"<li>{_escape(err)}</li>")
880
- text_lines.append(f"- {err}")
881
- html_lines.append("</ul>")
882
- html_lines.append("<br/>")
883
- text_lines.append("")
884
-
885
- return "\n".join(html_lines), "\n".join(text_lines), tg_total_views, vk_total_views, errors
886
-
887
-
888
- @app.get("/health")
889
- def health_check():
890
- return {"status": "ok"}
891
-
892
-
893
- @app.post("/parse", response_model=ParseResponse)
894
- def parse_links(payload: ParseRequest):
895
- raw_lines = normalize_links(payload.links)
896
- if not raw_lines:
897
- return ParseResponse(html="", text="", telegram_total=0, vk_total=0, errors=["Нет ссылок для обработки."])
898
-
899
- seen = set()
900
- lines: List[Tuple[str, str, object, object]] = []
901
- for link in raw_lines:
902
- plat = detect_platform(link)
903
- if plat == "telegram":
904
- canonical, username, mid = canonicalize_tg_link(link)
905
- if not canonical:
906
- canonical = link
907
- username = None
908
- mid = None
909
- if canonical in seen:
910
- continue
911
- seen.add(canonical)
912
- lines.append(("telegram", canonical, username, mid))
913
- elif plat == "vk":
914
- canonical, owner_id, post_id = canonicalize_vk_link(link)
915
- if not canonical:
916
- canonical = link
917
- if canonical in seen:
918
- continue
919
- seen.add(canonical)
920
- lines.append(("vk", canonical, owner_id, post_id))
921
- else:
922
- lines.append(("unknown", link, None, None))
923
-
924
- html, text, tg_total, vk_total, errors = build_output(lines)
925
- return ParseResponse(
926
- html=html,
927
- text=text,
928
- telegram_total=tg_total,
929
- vk_total=vk_total,
930
- errors=errors,
931
- )