Dink0 commited on
Commit
eb3170d
·
verified ·
1 Parent(s): 2cdbcd4

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -832
app.py DELETED
@@ -1,832 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from concurrent.futures import ThreadPoolExecutor
4
- from threading import Lock
5
- import time
6
- import os
7
- import re
8
- import html as html_lib
9
- from typing import List, Tuple, Union
10
-
11
- import requests
12
- from fastapi import FastAPI
13
- from fastapi.middleware.cors import CORSMiddleware
14
- from pydantic import BaseModel
15
-
16
- try:
17
- from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
18
- except Exception:
19
- PlaywrightTimeoutError = Exception
20
- sync_playwright = None
21
-
22
- APP_NAME = "pr-tool-backend"
23
-
24
- VK_API_VERSION = os.getenv("VK_API_VERSION", "5.131")
25
- VK_ACCESS_TOKEN = os.getenv("VK_ACCESS_TOKEN", "")
26
- REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "15"))
27
- MAX_WORKERS = int(os.getenv("MAX_WORKERS", "8"))
28
- VK_MAX_RETRIES = int(os.getenv("VK_MAX_RETRIES", "5"))
29
- VK_RETRY_DELAY = float(os.getenv("VK_RETRY_DELAY", "0.45"))
30
- VK_BATCH_SIZE = int(os.getenv("VK_BATCH_SIZE", "100"))
31
- PLAYWRIGHT_GOTO_TIMEOUT_MS = int(os.getenv("PLAYWRIGHT_GOTO_TIMEOUT_MS", "20000"))
32
- ENABLE_TELEGRAM_BROWSER_FALLBACK = os.getenv("ENABLE_TELEGRAM_BROWSER_FALLBACK", "1") != "0"
33
-
34
- _TELEGRAM_BROWSER_LOCK = Lock()
35
-
36
- app = FastAPI(title=APP_NAME)
37
-
38
- # Для простоты разрешаем все источники. Можно сузить список доменов в проде.
39
- app.add_middleware(
40
- CORSMiddleware,
41
- allow_origins=["*"],
42
- allow_credentials=False,
43
- allow_methods=["POST", "GET", "OPTIONS"],
44
- allow_headers=["*"]
45
- )
46
-
47
-
48
- class ParseRequest(BaseModel):
49
- links: Union[List[str], str]
50
-
51
-
52
- class ParseResponse(BaseModel):
53
- html: str
54
- text: str
55
- telegram_total: int
56
- vk_total: int
57
- errors: List[str]
58
-
59
-
60
- # ---------- Вспомогательные функции ----------
61
-
62
- def human_format_views(num: int) -> str:
63
- if num >= 1_000_000:
64
- value = num / 1_000_000
65
- suffix = "M"
66
- else:
67
- value = num / 1000
68
- suffix = "K"
69
-
70
- rounded = round(value, 1)
71
- if rounded.is_integer():
72
- formatted = f"{int(rounded)}{suffix}"
73
- else:
74
- formatted = f"{rounded:.1f}{suffix}"
75
-
76
- return formatted.replace(".", ",")
77
-
78
-
79
- def detect_platform(link: str) -> str:
80
- link = link.strip().lower()
81
- if "t.me/" in link or "telegram.me/" in link:
82
- return "telegram"
83
- if "vk.com/wall" in link or "vk.ru/wall" in link:
84
- return "vk"
85
- return ""
86
-
87
-
88
- def canonicalize_tg_link(link: str) -> Tuple[str | None, str | None, str | None]:
89
- link = link.strip()
90
- m = re.match(r"https?://(?:t(?:elegram)?\.me)/(?:s/)?([^/]+)/(?P<id>\d+)", link, re.IGNORECASE)
91
- if not m:
92
- return None, None, None
93
- username = m.group(1)
94
- message_id = m.group("id")
95
- canonical = f"https://t.me/{username}/{message_id}"
96
- return canonical, username, message_id
97
-
98
-
99
- def canonicalize_vk_link(link: str) -> Tuple[str | None, int | None, str | None]:
100
- link = link.strip()
101
- m = re.search(r"(?:vk\.com|vk\.ru)/wall(-?\d+)_(\d+)", link)
102
- if not m:
103
- return None, None, None
104
- owner_id = int(m.group(1))
105
- post_id = m.group(2)
106
- canonical = f"https://vk.com/wall{owner_id}_{post_id}"
107
- return canonical, owner_id, post_id
108
-
109
-
110
- def process_telegram_link(channel_username: str, message_id: str, canonical_link: str) -> Tuple[str, int]:
111
- # Набор альтернативных адресов (если t.me не резолвится)
112
- urls = [
113
- f"https://t.me/{channel_username}/{message_id}?embed=1",
114
- f"https://t.me/{channel_username}/{message_id}?embed=1&single=1",
115
- f"https://t.me/{channel_username}/{message_id}?embed=1&mode=tme",
116
- f"https://t.me/s/{channel_username}/{message_id}",
117
- f"https://t.me/{channel_username}/{message_id}",
118
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}?embed=1",
119
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}?embed=1&single=1",
120
- f"https://r.jina.ai/http://t.me/s/{channel_username}/{message_id}",
121
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}",
122
- ]
123
- headers = {"User-Agent": "Mozilla/5.0"}
124
- post_headers = {
125
- "User-Agent": "Mozilla/5.0",
126
- "Content-Type": "application/x-www-form-urlencoded",
127
- }
128
-
129
- def _parse_views_number(s: str) -> int:
130
- if not s:
131
- return 0
132
- s = s.strip().replace("\u00a0", "").replace(" ", "").replace(",", ".")
133
- m2 = re.match(r"([\d\.]+)\s*([kKmM]?)", s)
134
- if not m2:
135
- return 0
136
- val = float(m2.group(1))
137
- suf = m2.group(2).lower()
138
- if suf == "k":
139
- val *= 1_000
140
- elif suf == "m":
141
- val *= 1_000_000
142
- return int(val)
143
-
144
- def _fetch_ok_responses(url_list):
145
- texts: List[str] = []
146
- last_err = None
147
- for u in url_list:
148
- try:
149
- resp = requests.get(u, headers=headers, timeout=REQUEST_TIMEOUT)
150
- if resp.status_code == 200 and resp.text:
151
- texts.append(resp.text)
152
-
153
- # Встроенный tg-виджет сам делает POST на тот же URL с _rl=1.
154
- # Иногда именно этот ответ содержит уже дорисованный widget DOM с просмотрами.
155
- if "tgme_widget_message_views" not in resp.text:
156
- post_resp = requests.post(
157
- u,
158
- headers=post_headers,
159
- data="_rl=1",
160
- timeout=REQUEST_TIMEOUT,
161
- )
162
- if post_resp.status_code == 200 and post_resp.text:
163
- texts.append(post_resp.text)
164
- except Exception as exc:
165
- last_err = exc
166
- if texts:
167
- return texts
168
- raise last_err or Exception("Не удалось получить страницу Telegram")
169
-
170
- def _strip_emoji(s: str) -> str:
171
- # Убираем emoji/pictographs и variation selectors.
172
- s = re.sub(
173
- r"[\U0001F1E6-\U0001F1FF\U0001F300-\U0001FAFF\u2600-\u27BF\uFE0E\uFE0F]",
174
- "",
175
- s,
176
- )
177
- return re.sub(r"\s+", " ", s).strip()
178
-
179
- def _clean_title(raw: str, fallback: str) -> str:
180
- title = re.sub(
181
- r"<i\b[^>]*class=[\"'][^\"']*emoji[^\"']*[\"'][^>]*>.*?</i>",
182
- "",
183
- (raw or "").strip(),
184
- flags=re.IGNORECASE | re.DOTALL,
185
- )
186
- title = re.sub(r"<[^>]+>", "", title)
187
- title = html_lib.unescape(title)
188
- title = re.sub(r"\s*[-|]\s*Telegram\s*$", "", title, flags=re.IGNORECASE)
189
- title = _strip_emoji(title)
190
- return title or fallback
191
-
192
- def _is_placeholder_title(raw: str | None) -> bool:
193
- if not raw:
194
- return True
195
- normalized = _clean_title(raw, "").strip().lower()
196
- placeholders = {
197
- "",
198
- "telegram",
199
- "telegram widget",
200
- "telegram - a new era of messaging",
201
- "telegram – a new era of messaging",
202
- }
203
- if normalized in placeholders:
204
- return True
205
- if normalized.startswith("telegram:"):
206
- return True
207
- if "a new era of messaging" in normalized:
208
- return True
209
- return False
210
-
211
- def _is_username_like_title(raw: str | None) -> bool:
212
- if not raw:
213
- return False
214
- normalized = _clean_title(raw, "").strip().lower()
215
- username_variants = {
216
- channel_username.lower(),
217
- f"@{channel_username.lower()}",
218
- f"t.me/{channel_username.lower()}",
219
- }
220
- return normalized in username_variants
221
-
222
- def _extract_views(html: str) -> int:
223
- patterns = [
224
- # Виджет Telegram
225
- r'class="tgme_widget_message_views[^"]*">([\d\s\.,kKmM]+)<',
226
- # Резервный вариант из meta-description, но только если рядом явно "views"
227
- r'content="[^"]*?([\d\s\.,kKmM]+)\s+views[^"]*"',
228
- r'content="[^"]*?(?:Просмотры|просмотры|просмотров)[:\s]*([\d\s\.,kKmM]+)[^"]*"',
229
- # Текстовый fallback (например, через jina)
230
- r'(?:^|\n)\s*(?:Views|views|Просмотры|просмотры|Просмотров|просмотров)\s*[:\-]?\s*([\d\s\.,kKmM]+)\b',
231
- r'([\d\s\.,kKmM]+)\s*(?:views|Views|просмотров|Просмотров)\b',
232
- r'(?:👁|👀)\s*([\d\s\.,kKmM]+)\b',
233
- ]
234
- for pattern in patterns:
235
- m = re.search(pattern, html, re.IGNORECASE)
236
- if m:
237
- parsed = _parse_views_number(m.group(1))
238
- if parsed > 0:
239
- return parsed
240
- return 0
241
-
242
- def _extract_text_title(raw_text: str) -> str | None:
243
- looks_like_html = bool(re.search(r"<(?:!DOCTYPE|html|head|body|meta|script|div)\b", raw_text, re.IGNORECASE))
244
- patterns = [
245
- r'(?:^|\n)\s*Title:\s*(.+)',
246
- r'(?:^|\n)\s*Channel:\s*(.+)',
247
- r'(?:^|\n)\s*#\s+(.+)',
248
- ]
249
- for pattern in patterns:
250
- m = re.search(pattern, raw_text, re.IGNORECASE)
251
- if m:
252
- return m.group(1).strip()
253
-
254
- if looks_like_html:
255
- return None
256
-
257
- for line in raw_text.splitlines():
258
- line = line.strip()
259
- if not line:
260
- continue
261
- if line.startswith("http://") or line.startswith("https://"):
262
- continue
263
- if "views" in line.lower() or "просмотр" in line.lower():
264
- continue
265
- if _is_username_like_title(line):
266
- continue
267
- if len(line) <= 120:
268
- return line
269
- return None
270
-
271
- def _extract_title(raw_text: str) -> str | None:
272
- candidates: List[str] = []
273
-
274
- m_title_meta = re.search(
275
- r"<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
276
- raw_text,
277
- re.IGNORECASE,
278
- )
279
- if m_title_meta:
280
- candidates.append(m_title_meta.group(1).strip())
281
-
282
- m_twitter_title = re.search(
283
- r"<meta[^>]*property=[\"']twitter:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
284
- raw_text,
285
- re.IGNORECASE,
286
- )
287
- if m_twitter_title:
288
- candidates.append(m_twitter_title.group(1).strip())
289
-
290
- m_site_name = re.search(
291
- r"<meta[^>]*property=[\"']og:site_name[\"'][^>]*content=[\"']([^\"']+)[\"']",
292
- raw_text,
293
- re.IGNORECASE,
294
- )
295
- if m_site_name:
296
- candidates.append(m_site_name.group(1).strip())
297
-
298
- m_author = re.search(
299
- r'class="tgme_widget_message_author_name".*?<span[^>]*>(.*?)</span>',
300
- raw_text,
301
- re.IGNORECASE | re.DOTALL,
302
- )
303
- if m_author:
304
- candidates.append(m_author.group(1).strip())
305
-
306
- m_owner = re.search(
307
- r'class="tgme_widget_message_owner_name".*?<span[^>]*>(.*?)</span>',
308
- raw_text,
309
- re.IGNORECASE | re.DOTALL,
310
- )
311
- if m_owner:
312
- candidates.append(m_owner.group(1).strip())
313
-
314
- text_title = _extract_text_title(raw_text)
315
- if text_title:
316
- candidates.append(text_title)
317
-
318
- for candidate in candidates:
319
- if not _is_placeholder_title(candidate):
320
- return candidate
321
- return None
322
-
323
- def _pick_widget_html(raw_text: str) -> str | None:
324
- candidates = [raw_text]
325
- iframe_matches = re.findall(r"<iframe\b[^>]*>(.*?)</iframe>", raw_text, re.IGNORECASE | re.DOTALL)
326
- candidates.extend(iframe_matches)
327
-
328
- for candidate in candidates:
329
- if (
330
- "tgme_widget_message_views" in candidate
331
- or "tgme_widget_message_owner_name" in candidate
332
- or "tgme_widget_message_author_name" in candidate
333
- ):
334
- return candidate
335
- return None
336
-
337
- def _fetch_browser_response() -> str | None:
338
- if not ENABLE_TELEGRAM_BROWSER_FALLBACK or sync_playwright is None:
339
- return None
340
-
341
- browser_urls = [
342
- f"https://t.me/{channel_username}/{message_id}?embed=1&single=1",
343
- f"https://t.me/{channel_username}/{message_id}?embed=1",
344
- f"https://t.me/{channel_username}/{message_id}",
345
- ]
346
-
347
- with _TELEGRAM_BROWSER_LOCK:
348
- try:
349
- with sync_playwright() as playwright:
350
- browser = playwright.chromium.launch(
351
- headless=True,
352
- args=["--no-sandbox", "--disable-dev-shm-usage"],
353
- )
354
- context = browser.new_context(
355
- user_agent=headers["User-Agent"],
356
- viewport={"width": 1280, "height": 900},
357
- )
358
- page = context.new_page()
359
-
360
- try:
361
- for url in browser_urls:
362
- try:
363
- page.goto(url, wait_until="domcontentloaded", timeout=PLAYWRIGHT_GOTO_TIMEOUT_MS)
364
- except PlaywrightTimeoutError:
365
- pass
366
- except Exception:
367
- continue
368
-
369
- try:
370
- page.wait_for_timeout(700)
371
- page.wait_for_selector(
372
- ".tgme_widget_message_views, .tgme_widget_message_owner_name, .tgme_widget_message_author_name",
373
- timeout=4000,
374
- )
375
- except PlaywrightTimeoutError:
376
- pass
377
- except Exception:
378
- pass
379
-
380
- frame_htmls: List[str] = []
381
- try:
382
- frame_htmls.append(page.content())
383
- except Exception:
384
- pass
385
-
386
- for frame in page.frames:
387
- if frame == page.main_frame:
388
- continue
389
- try:
390
- frame_htmls.append(frame.content())
391
- except Exception:
392
- continue
393
-
394
- for frame_html in frame_htmls:
395
- widget_html = _pick_widget_html(frame_html)
396
- if widget_html:
397
- return widget_html
398
- finally:
399
- context.close()
400
- browser.close()
401
- except Exception:
402
- return None
403
-
404
- return None
405
-
406
- try:
407
- responses = _fetch_ok_responses(urls)
408
-
409
- title_raw = None
410
- views = 0
411
- for response_text in responses:
412
- candidate_title = _extract_title(response_text)
413
- if candidate_title:
414
- if not title_raw:
415
- title_raw = candidate_title
416
- elif _is_username_like_title(title_raw) and not _is_username_like_title(candidate_title):
417
- title_raw = candidate_title
418
-
419
- candidate_views = _extract_views(response_text)
420
- if candidate_views > views:
421
- views = candidate_views
422
-
423
- if title_raw and not _is_username_like_title(title_raw) and views > 0:
424
- break
425
-
426
- if not title_raw or _is_username_like_title(title_raw) or views <= 0:
427
- browser_response = _fetch_browser_response()
428
- if browser_response:
429
- browser_title = _extract_title(browser_response)
430
- browser_views = _extract_views(browser_response)
431
- if browser_title and (not title_raw or _is_username_like_title(title_raw)):
432
- title_raw = browser_title
433
- if browser_views > views:
434
- views = browser_views
435
-
436
- title = _clean_title(title_raw or "", channel_username)
437
-
438
- return title, views
439
-
440
- except Exception as exc:
441
- return f"Ошибка (TG) при обработке {canonical_link}: {exc}", 0
442
-
443
-
444
- def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[str, int]:
445
- posts_param = f"{owner_id}_{post_id}"
446
- api_url = "https://api.vk.com/method/wall.getById"
447
-
448
- for attempt in range(1, VK_MAX_RETRIES + 1):
449
- params = {
450
- "posts": posts_param,
451
- "v": VK_API_VERSION,
452
- "extended": 1,
453
- }
454
- if VK_ACCESS_TOKEN:
455
- params["access_token"] = VK_ACCESS_TOKEN
456
-
457
- try:
458
- resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
459
- except Exception as exc:
460
- return f"**Ошибка (VK)**: {exc} — {canonical_link}", 0
461
-
462
- if resp.status_code != 200:
463
- return f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical_link}", 0
464
-
465
- data = resp.json()
466
- if "error" in data:
467
- error = data["error"]
468
- error_msg = error.get("error_msg", "")
469
- error_code = error.get("error_code")
470
- if error_code == 6 or "too many requests per second" in error_msg.lower():
471
- time.sleep(VK_RETRY_DELAY * attempt)
472
- continue
473
- return f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical_link}", 0
474
- break
475
- else:
476
- return f"**Ошибка (VK)**: Too many requests per second — {canonical_link}", 0
477
-
478
- response_data = data.get("response", {})
479
- items = response_data.get("items", [])
480
- if not items:
481
- return f"**Ошибка (VK)**: пост не найден — {canonical_link}", 0
482
-
483
- post = items[0]
484
- views = post.get("views", {}).get("count", 0)
485
- post_owner_id = post.get("owner_id", owner_id)
486
-
487
- title = None
488
- if post_owner_id < 0:
489
- group_id = -post_owner_id
490
- for group in response_data.get("groups", []):
491
- if group.get("id") == group_id:
492
- title = group.get("name")
493
- break
494
- else:
495
- user_id = post_owner_id
496
- for profile in response_data.get("profiles", []):
497
- if profile.get("id") == user_id:
498
- first_name = profile.get("first_name", "")
499
- last_name = profile.get("last_name", "")
500
- title = (first_name + " " + last_name).strip()
501
- break
502
- if not title:
503
- title = "VK пост"
504
-
505
- return title, views
506
-
507
-
508
- def process_vk_batch(
509
- batch_items: List[Tuple[str, int, str]]
510
- ) -> dict[str, Tuple[str, int]]:
511
- api_url = "https://api.vk.com/method/wall.getById"
512
- posts = ",".join(f"{owner_id}_{post_id}" for _, owner_id, post_id in batch_items)
513
-
514
- for attempt in range(1, VK_MAX_RETRIES + 1):
515
- params = {
516
- "posts": posts,
517
- "v": VK_API_VERSION,
518
- "extended": 1,
519
- }
520
- if VK_ACCESS_TOKEN:
521
- params["access_token"] = VK_ACCESS_TOKEN
522
-
523
- try:
524
- resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
525
- except Exception as exc:
526
- return {
527
- canonical: (f"**Ошибка (VK)**: {exc} — {canonical}", 0)
528
- for canonical, _, _ in batch_items
529
- }
530
-
531
- if resp.status_code != 200:
532
- return {
533
- canonical: (f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical}", 0)
534
- for canonical, _, _ in batch_items
535
- }
536
-
537
- data = resp.json()
538
- if "error" in data:
539
- error = data["error"]
540
- error_msg = error.get("error_msg", "")
541
- error_code = error.get("error_code")
542
- if error_code == 6 or "too many requests per second" in error_msg.lower():
543
- time.sleep(VK_RETRY_DELAY * attempt)
544
- continue
545
- return {
546
- canonical: (f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical}", 0)
547
- for canonical, _, _ in batch_items
548
- }
549
- break
550
- else:
551
- return {
552
- canonical: (f"**Ошибка (VK)**: Too many requests per second — {canonical}", 0)
553
- for canonical, _, _ in batch_items
554
- }
555
-
556
- response_data = data.get("response", {})
557
- items = response_data.get("items", [])
558
- groups = {group.get("id"): group for group in response_data.get("groups", [])}
559
- profiles = {profile.get("id"): profile for profile in response_data.get("profiles", [])}
560
- item_map = {
561
- f"{item.get('owner_id')}_{item.get('id')}": item
562
- for item in items
563
- if item.get("owner_id") is not None and item.get("id") is not None
564
- }
565
-
566
- result: dict[str, Tuple[str, int]] = {}
567
- for canonical, owner_id, post_id in batch_items:
568
- key = f"{owner_id}_{post_id}"
569
- post = item_map.get(key)
570
- if not post:
571
- result[canonical] = (f"**Ошибка (VK)**: пост не найден — {canonical}", 0)
572
- continue
573
-
574
- views = post.get("views", {}).get("count", 0)
575
- post_owner_id = post.get("owner_id", owner_id)
576
- title = None
577
-
578
- if post_owner_id < 0:
579
- group = groups.get(-post_owner_id)
580
- if group:
581
- title = group.get("name")
582
- else:
583
- profile = profiles.get(post_owner_id)
584
- if profile:
585
- first_name = profile.get("first_name", "")
586
- last_name = profile.get("last_name", "")
587
- title = (first_name + " " + last_name).strip()
588
-
589
- result[canonical] = (title or "VK пост", views)
590
-
591
- return result
592
-
593
-
594
- def normalize_links(links: Union[List[str], str]) -> List[str]:
595
- if isinstance(links, str):
596
- raw = links.splitlines()
597
- else:
598
- raw = []
599
- for item in links:
600
- raw.extend(str(item).splitlines())
601
- return [line.strip() for line in raw if line.strip()]
602
-
603
-
604
- def _escape(text: str) -> str:
605
- return html_lib.escape(text, quote=True)
606
-
607
-
608
- def resolve_line_item(line: Tuple[str, str, object, object]) -> Tuple[str, str, str | None, int]:
609
- plat, canonical, a, b = line
610
-
611
- if plat == "telegram":
612
- username, mid = a, b
613
- if not username or not mid:
614
- return plat, canonical, "Telegram", 0
615
- title, views = process_telegram_link(username, mid, canonical)
616
- return plat, canonical, title, views
617
-
618
- if plat == "vk":
619
- owner_id, post_id = a, b
620
- if owner_id is None or post_id is None:
621
- return plat, canonical, "VK пост", 0
622
- title, views = process_vk_link(owner_id, post_id, canonical)
623
- return plat, canonical, title, views
624
-
625
- return plat, canonical, None, 0
626
-
627
-
628
- def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str, int, int, List[str]]:
629
- errors: List[str] = []
630
-
631
- tg_groups = {}
632
- vk_groups = {}
633
-
634
- telegram_lines = [line for line in lines if line[0] == "telegram"]
635
- vk_lines = [line for line in lines if line[0] == "vk"]
636
- telegram_results = {}
637
- vk_results = {}
638
-
639
- if telegram_lines:
640
- worker_count = max(1, min(MAX_WORKERS, len(telegram_lines)))
641
- with ThreadPoolExecutor(max_workers=worker_count) as executor:
642
- for result in executor.map(resolve_line_item, telegram_lines):
643
- telegram_results[result[1]] = result
644
-
645
- valid_vk_batch: List[Tuple[str, int, str]] = []
646
- for _, canonical, owner_id, post_id in vk_lines:
647
- if owner_id is not None and post_id is not None:
648
- valid_vk_batch.append((canonical, owner_id, post_id))
649
-
650
- if valid_vk_batch:
651
- batch_size = max(1, VK_BATCH_SIZE)
652
- for start in range(0, len(valid_vk_batch), batch_size):
653
- chunk = valid_vk_batch[start:start + batch_size]
654
- vk_results.update(process_vk_batch(chunk))
655
-
656
- for plat, canonical, a, b in lines:
657
- if plat == "telegram":
658
- username, mid = a, b
659
- if not username or not mid:
660
- key = canonical
661
- tg_groups.setdefault(key, {"title": "Telegram", "items": []})
662
- tg_groups[key]["items"].append((canonical, 0))
663
- else:
664
- _, _, title, views = telegram_results.get(canonical, (plat, canonical, username, 0))
665
- key = username
666
- if key not in tg_groups:
667
- tg_groups[key] = {"title": title, "items": []}
668
- if not tg_groups[key].get("title") or str(tg_groups[key]["title"]).startswith("**Ошибка"):
669
- tg_groups[key]["title"] = title
670
- tg_groups[key]["items"].append((canonical, views))
671
-
672
- elif plat == "vk":
673
- owner_id, post_id = a, b
674
- if owner_id is None or post_id is None:
675
- key = canonical
676
- vk_groups.setdefault(key, {"title": "VK пост", "items": []})
677
- vk_groups[key]["items"].append((canonical, 0))
678
- else:
679
- title, views = vk_results.get(canonical, process_vk_link(owner_id, post_id, canonical))
680
- key = str(owner_id)
681
- if key not in vk_groups:
682
- vk_groups[key] = {"title": title, "items": []}
683
- if not vk_groups[key].get("title") or str(vk_groups[key]["title"]).startswith("**Ошибка"):
684
- vk_groups[key]["title"] = title
685
- vk_groups[key]["items"].append((canonical, views))
686
- else:
687
- errors.append(f"Неизвестная платформа: {canonical}")
688
-
689
- tg_total_views = sum(v for g in tg_groups.values() for _, v in g["items"])
690
- vk_total_views = sum(v for g in vk_groups.values() for _, v in g["items"])
691
-
692
- tg_sorted = sorted(
693
- tg_groups.items(),
694
- key=lambda kv: sum(v for _, v in kv[1]["items"]),
695
- reverse=True,
696
- )
697
- vk_sorted = sorted(
698
- vk_groups.items(),
699
- key=lambda kv: sum(v for _, v in kv[1]["items"]),
700
- reverse=True,
701
- )
702
-
703
- html_lines: List[str] = []
704
- text_lines: List[str] = []
705
-
706
- # --- Telegram ---
707
- html_lines.append("<h2>Telegram</h2>")
708
- html_lines.append(f'Суммарно посты собрали <b>{human_format_views(tg_total_views)}</b> просмотров.')
709
- text_lines.append("Telegram")
710
- text_lines.append(f"Суммарно посты собрали {human_format_views(tg_total_views)} просмотров.")
711
- if tg_sorted:
712
- html_lines.append("<ol>")
713
- for idx, (_, data) in enumerate(tg_sorted, start=1):
714
- title = data["title"] or "Telegram"
715
- items = data["items"]
716
- first_link, _first_views = items[0]
717
-
718
- title_html = _escape(title)
719
- first_link_html = _escape(first_link)
720
- line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
721
- if len(items) > 1:
722
- for link2, _v in items[1:]:
723
- line_html += f' + <a href="{_escape(link2)}">ещё</a>'
724
- views_str = " + ".join(human_format_views(v) for _, v in items)
725
- line_html += f" — {views_str}</li>"
726
- html_lines.append(line_html)
727
-
728
- line_text = f"{idx}. {title} ({first_link})"
729
- if len(items) > 1:
730
- extra_links = ", ".join(link2 for link2, _v in items[1:])
731
- line_text += f" + ещё: {extra_links}"
732
- line_text += f" — {views_str}"
733
- text_lines.append(line_text)
734
- html_lines.append("</ol>")
735
- else:
736
- html_lines.append("<i>Нет ссылок на Telegram</i>")
737
- text_lines.append("Нет ссылок на Telegram")
738
- html_lines.append("<br/>")
739
- text_lines.append("")
740
-
741
- # --- VK ---
742
- html_lines.append("<h2>ВКонтакте</h2>")
743
- html_lines.append(f'Суммарно посты собрали <b>{human_format_views(vk_total_views)}</b> просмотров.')
744
- text_lines.append("ВКонтакте")
745
- text_lines.append(f"Суммарно посты собрали {human_format_views(vk_total_views)} просмотров.")
746
- if vk_sorted:
747
- html_lines.append("<ol>")
748
- for idx, (_, data) in enumerate(vk_sorted, start=1):
749
- title = data["title"] or "VK пост"
750
- items = data["items"]
751
- first_link, _first_views = items[0]
752
-
753
- title_html = _escape(title)
754
- first_link_html = _escape(first_link)
755
- line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
756
- if len(items) > 1:
757
- for link2, _v in items[1:]:
758
- line_html += f' + <a href="{_escape(link2)}">ещё</a>'
759
- views_str = " + ".join(human_format_views(v) for _, v in items)
760
- line_html += f" — {views_str}</li>"
761
- html_lines.append(line_html)
762
-
763
- line_text = f"{idx}. {title} ({first_link})"
764
- if len(items) > 1:
765
- extra_links = ", ".join(link2 for link2, _v in items[1:])
766
- line_text += f" + ещё: {extra_links}"
767
- line_text += f" — {views_str}"
768
- text_lines.append(line_text)
769
- html_lines.append("</ol>")
770
- else:
771
- html_lines.append("<i>Нет ссылок на ВКонтакте</i>")
772
- text_lines.append("Нет ссылок на ВКонтакте")
773
- html_lines.append("<br/>")
774
- text_lines.append("")
775
-
776
- if errors:
777
- html_lines.append("<h2>Ошибки</h2>")
778
- html_lines.append("<ul>")
779
- for err in errors:
780
- html_lines.append(f"<li>{_escape(err)}</li>")
781
- text_lines.append(f"- {err}")
782
- html_lines.append("</ul>")
783
- html_lines.append("<br/>")
784
- text_lines.append("")
785
-
786
- return "\n".join(html_lines), "\n".join(text_lines), tg_total_views, vk_total_views, errors
787
-
788
-
789
- @app.get("/health")
790
- def health_check():
791
- return {"status": "ok"}
792
-
793
-
794
- @app.post("/parse", response_model=ParseResponse)
795
- def parse_links(payload: ParseRequest):
796
- raw_lines = normalize_links(payload.links)
797
- if not raw_lines:
798
- return ParseResponse(html="", text="", telegram_total=0, vk_total=0, errors=["Нет ссылок для обработки."])
799
-
800
- seen = set()
801
- lines: List[Tuple[str, str, object, object]] = []
802
- for link in raw_lines:
803
- plat = detect_platform(link)
804
- if plat == "telegram":
805
- canonical, username, mid = canonicalize_tg_link(link)
806
- if not canonical:
807
- canonical = link
808
- username = None
809
- mid = None
810
- if canonical in seen:
811
- continue
812
- seen.add(canonical)
813
- lines.append(("telegram", canonical, username, mid))
814
- elif plat == "vk":
815
- canonical, owner_id, post_id = canonicalize_vk_link(link)
816
- if not canonical:
817
- canonical = link
818
- if canonical in seen:
819
- continue
820
- seen.add(canonical)
821
- lines.append(("vk", canonical, owner_id, post_id))
822
- else:
823
- lines.append(("unknown", link, None, None))
824
-
825
- html, text, tg_total, vk_total, errors = build_output(lines)
826
- return ParseResponse(
827
- html=html,
828
- text=text,
829
- telegram_total=tg_total,
830
- vk_total=vk_total,
831
- errors=errors,
832
- )