Dink0 Claude Opus 4.8 commited on
Commit
397f7dc
·
1 Parent(s): b751754

Update app.py with new version

Browse files

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (1) hide show
  1. app.py +404 -245
app.py CHANGED
@@ -1,13 +1,12 @@
1
  from __future__ import annotations
2
 
3
- from concurrent.futures import ThreadPoolExecutor
4
- import asyncio
5
- from threading import Lock
6
  import time
7
  import os
8
  import re
9
  import html as html_lib
10
- from typing import List, Tuple, Union
11
 
12
  import requests
13
  from fastapi import FastAPI
@@ -21,12 +20,6 @@ except Exception:
21
  TelegramClient = None
22
  StringSession = None
23
 
24
- try:
25
- from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
26
- except Exception:
27
- PlaywrightTimeoutError = Exception
28
- sync_playwright = None
29
-
30
  APP_NAME = "pr-tool-backend"
31
 
32
  VK_API_VERSION = os.getenv("VK_API_VERSION", "5.131")
@@ -34,16 +27,34 @@ VK_ACCESS_TOKEN = os.getenv("VK_ACCESS_TOKEN", "")
34
  TELEGRAM_API_ID = int(os.getenv("TELEGRAM_API_ID", "0") or "0")
35
  TELEGRAM_API_HASH = os.getenv("TELEGRAM_API_HASH", "")
36
  TELEGRAM_STRING_SESSION = os.getenv("TELEGRAM_STRING_SESSION", "")
37
- REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "15"))
38
- TELEGRAM_OP_TIMEOUT = float(os.getenv("TELEGRAM_OP_TIMEOUT", "8"))
39
- TELEGRAM_CONCURRENCY = int(os.getenv("TELEGRAM_CONCURRENCY", "6"))
 
 
 
 
 
 
 
 
 
 
40
  VK_MAX_RETRIES = int(os.getenv("VK_MAX_RETRIES", "5"))
41
  VK_RETRY_DELAY = float(os.getenv("VK_RETRY_DELAY", "0.45"))
42
  VK_BATCH_SIZE = int(os.getenv("VK_BATCH_SIZE", "100"))
43
- PLAYWRIGHT_GOTO_TIMEOUT_MS = int(os.getenv("PLAYWRIGHT_GOTO_TIMEOUT_MS", "20000"))
44
- ENABLE_TELEGRAM_BROWSER_FALLBACK = os.getenv("ENABLE_TELEGRAM_BROWSER_FALLBACK", "1") != "0"
45
 
46
- _TELEGRAM_BROWSER_LOCK = Lock()
 
 
 
 
 
 
 
 
 
 
47
 
48
  app = FastAPI(title=APP_NAME)
49
 
@@ -53,10 +64,30 @@ app.add_middleware(
53
  allow_origins=["*"],
54
  allow_credentials=False,
55
  allow_methods=["POST", "GET", "OPTIONS"],
56
- allow_headers=["*"]
57
  )
58
 
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  class ParseRequest(BaseModel):
61
  links: Union[List[str], str]
62
 
@@ -67,6 +98,7 @@ class ParseResponse(BaseModel):
67
  telegram_total: int
68
  vk_total: int
69
  errors: List[str]
 
70
 
71
 
72
  # ---------- Вспомогательные функции ----------
@@ -89,6 +121,16 @@ def clean_telegram_title(raw: str, fallback: str) -> str:
89
  title = re.sub(r"\s+", " ", title).strip()
90
  return title or fallback
91
 
 
 
 
 
 
 
 
 
 
 
92
  def human_format_views(num: int) -> str:
93
  if num >= 1_000_000:
94
  value = num / 1_000_000
@@ -106,16 +148,34 @@ def human_format_views(num: int) -> str:
106
  return formatted.replace(".", ",")
107
 
108
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
  def detect_platform(link: str) -> str:
110
  link = link.strip().lower()
111
  if "t.me/" in link or "telegram.me/" in link:
112
  return "telegram"
113
  if "vk.com/wall" in link or "vk.ru/wall" in link:
114
  return "vk"
 
 
115
  return ""
116
 
117
 
118
- def canonicalize_tg_link(link: str) -> Tuple[str | None, str | None, str | None]:
119
  link = link.strip()
120
  private_match = re.match(r"https?://(?:t(?:elegram)?\.me)/c/(\d+)/(\d+)", link, re.IGNORECASE)
121
  if private_match:
@@ -129,7 +189,7 @@ def canonicalize_tg_link(link: str) -> Tuple[str | None, str | None, str | None]
129
  return canonical, username, message_id
130
 
131
 
132
- def canonicalize_vk_link(link: str) -> Tuple[str | None, int | None, str | None]:
133
  link = link.strip()
134
  m = re.search(r"(?:vk\.com|vk\.ru)/wall(-?\d+)_(\d+)", link)
135
  if not m:
@@ -140,6 +200,17 @@ def canonicalize_vk_link(link: str) -> Tuple[str | None, int | None, str | None]
140
  return canonical, owner_id, post_id
141
 
142
 
 
 
 
 
 
 
 
 
 
 
 
143
  def _telethon_ready() -> bool:
144
  return bool(
145
  TelegramClient
@@ -150,125 +221,112 @@ def _telethon_ready() -> bool:
150
  )
151
 
152
 
153
- def process_telegram_batch(batch_items: List[Tuple[str, str, str]]) -> dict[str, Tuple[str, int]]:
154
- if not batch_items:
155
- return {}
156
- results: dict[str, Tuple[str, int]] = {}
157
- worker_count = max(1, min(8, len(batch_items)))
158
-
159
- def _resolve(item: Tuple[str, str, str]) -> Tuple[str, Tuple[str, int]]:
160
- canonical, username, message_id = item
161
- return canonical, process_telegram_link(username, message_id, canonical)
162
-
163
- with ThreadPoolExecutor(max_workers=worker_count) as executor:
164
- for canonical, result in executor.map(_resolve, batch_items):
165
- results[canonical] = result
166
-
167
- return results
168
-
169
 
170
  def process_telegram_link(channel_username: str, message_id: str, canonical_link: str) -> Tuple[str, int]:
171
- urls = [
172
  f"https://t.me/{channel_username}/{message_id}?embed=1",
173
  f"https://t.me/s/{channel_username}/{message_id}",
174
- f"https://telegram.me/{channel_username}/{message_id}?embed=1",
175
- f"https://r.jina.ai/http://t.me/{channel_username}/{message_id}?embed=1",
176
- f"https://r.jina.ai/http://t.me/s/{channel_username}/{message_id}",
177
- f"https://r.jina.ai/http://telegram.me/{channel_username}/{message_id}?embed=1",
178
  ]
179
- headers = {"User-Agent": "Mozilla/5.0"}
180
-
181
- def _parse_views_number(raw: str) -> int:
182
- if not raw:
183
- return 0
184
- normalized = raw.strip().replace("\u00a0", "").replace(" ", "").replace(",", ".")
185
- match = re.match(r"([\d\.]+)\s*([kKmM]?)", normalized)
186
- if not match:
187
- return 0
188
- value = float(match.group(1))
189
- suffix = match.group(2).lower()
190
- if suffix == "k":
191
- value *= 1_000
192
- elif suffix == "m":
193
- value *= 1_000_000
194
- return int(value)
195
-
196
- def _fetch_first_ok(url_list: List[str]) -> str:
197
- last_err = None
198
- for url in url_list:
199
- try:
200
- resp = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT)
201
- if resp.status_code == 200 and resp.text:
202
- return resp.text
203
- except Exception as exc:
204
- last_err = exc
205
- raise last_err or Exception("Не удалось получить страницу Telegram")
206
 
207
- try:
208
- page_html = _fetch_first_ok(urls)
209
 
210
- title = None
211
- meta_title = re.search(
212
- r"<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
 
 
 
 
 
 
 
 
213
  page_html,
214
- re.IGNORECASE,
215
  )
216
- if meta_title:
217
- title = meta_title.group(1).strip()
218
- if not title:
219
- owner_title = re.search(
220
- r'class="tgme_widget_message_owner_name".*?<span[^>]*>(.*?)</span>',
221
- page_html,
222
- re.IGNORECASE | re.DOTALL,
223
- )
224
- if owner_title:
225
- title = owner_title.group(1).strip()
226
- title = clean_telegram_title(title or "", channel_username)
227
-
228
- views = 0
229
- widget_views = re.search(
230
- r'class="tgme_widget_message_views[^"]*">([\d\s\.,kKmM]+)<',
231
  page_html,
232
  re.IGNORECASE,
233
  )
234
- if widget_views:
235
- views = _parse_views_number(widget_views.group(1))
236
- else:
237
- candidates = re.findall(
238
- r"(\d[\d\s\.,]*)([kKmM]?)\s*(?:views|просмотр|просмотра|просмотров|переглядів|visualizações|visualizzazioni|ansichten|visninger|visitas)?",
 
 
 
 
 
 
 
 
 
 
 
 
 
239
  page_html,
240
  re.IGNORECASE,
241
  )
242
- if candidates:
243
- last_num, last_suffix = candidates[-1]
244
- views = _parse_views_number(last_num + last_suffix)
245
-
246
- return title, views
247
- except Exception as exc:
248
- return f"Ошибка (TG) при обработке {canonical_link}: {exc}", 0
249
 
250
 
251
  def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[str, int]:
 
252
  posts_param = f"{owner_id}_{post_id}"
253
  api_url = "https://api.vk.com/method/wall.getById"
254
 
 
255
  for attempt in range(1, VK_MAX_RETRIES + 1):
256
- params = {
257
- "posts": posts_param,
258
- "v": VK_API_VERSION,
259
- "extended": 1,
260
- }
261
  if VK_ACCESS_TOKEN:
262
  params["access_token"] = VK_ACCESS_TOKEN
263
-
264
  try:
265
  resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
266
- except Exception as exc:
267
  return f"**Ошибка (VK)**: {exc} — {canonical_link}", 0
268
-
269
  if resp.status_code != 200:
270
  return f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical_link}", 0
271
-
272
  data = resp.json()
273
  if "error" in data:
274
  error = data["error"]
@@ -282,7 +340,7 @@ def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[s
282
  else:
283
  return f"**Ошибка (VK)**: Too many requests per second — {canonical_link}", 0
284
 
285
- response_data = data.get("response", {})
286
  items = response_data.get("items", [])
287
  if not items:
288
  return f"**Ошибка (VK)**: пост не найден — {canonical_link}", 0
@@ -299,47 +357,31 @@ def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[s
299
  title = group.get("name")
300
  break
301
  else:
302
- user_id = post_owner_id
303
  for profile in response_data.get("profiles", []):
304
- if profile.get("id") == user_id:
305
- first_name = profile.get("first_name", "")
306
- last_name = profile.get("last_name", "")
307
- title = (first_name + " " + last_name).strip()
308
  break
309
  if not title:
310
  title = "VK пост"
311
-
312
  return title, views
313
 
314
 
315
- def process_vk_batch(
316
- batch_items: List[Tuple[str, int, str]]
317
- ) -> dict[str, Tuple[str, int]]:
318
  api_url = "https://api.vk.com/method/wall.getById"
319
  posts = ",".join(f"{owner_id}_{post_id}" for _, owner_id, post_id in batch_items)
320
 
 
321
  for attempt in range(1, VK_MAX_RETRIES + 1):
322
- params = {
323
- "posts": posts,
324
- "v": VK_API_VERSION,
325
- "extended": 1,
326
- }
327
  if VK_ACCESS_TOKEN:
328
  params["access_token"] = VK_ACCESS_TOKEN
329
-
330
  try:
331
  resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
332
- except Exception as exc:
333
- return {
334
- canonical: (f"**Ошибка (VK)**: {exc} — {canonical}", 0)
335
- for canonical, _, _ in batch_items
336
- }
337
 
338
  if resp.status_code != 200:
339
- return {
340
- canonical: (f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical}", 0)
341
- for canonical, _, _ in batch_items
342
- }
343
 
344
  data = resp.json()
345
  if "error" in data:
@@ -349,18 +391,12 @@ def process_vk_batch(
349
  if error_code == 6 or "too many requests per second" in error_msg.lower():
350
  time.sleep(VK_RETRY_DELAY * attempt)
351
  continue
352
- return {
353
- canonical: (f"**Ошибка (VK)**: {error_msg or 'API error'} — {canonical}", 0)
354
- for canonical, _, _ in batch_items
355
- }
356
  break
357
  else:
358
- return {
359
- canonical: (f"**Ошибка (VK)**: Too many requests per second — {canonical}", 0)
360
- for canonical, _, _ in batch_items
361
- }
362
 
363
- response_data = data.get("response", {})
364
  items = response_data.get("items", [])
365
  groups = {group.get("id"): group for group in response_data.get("groups", [])}
366
  profiles = {profile.get("id"): profile for profile in response_data.get("profiles", [])}
@@ -370,7 +406,7 @@ def process_vk_batch(
370
  if item.get("owner_id") is not None and item.get("id") is not None
371
  }
372
 
373
- result: dict[str, Tuple[str, int]] = {}
374
  for canonical, owner_id, post_id in batch_items:
375
  key = f"{owner_id}_{post_id}"
376
  post = item_map.get(key)
@@ -381,7 +417,6 @@ def process_vk_batch(
381
  views = post.get("views", {}).get("count", 0)
382
  post_owner_id = post.get("owner_id", owner_id)
383
  title = None
384
-
385
  if post_owner_id < 0:
386
  group = groups.get(-post_owner_id)
387
  if group:
@@ -389,15 +424,91 @@ def process_vk_batch(
389
  else:
390
  profile = profiles.get(post_owner_id)
391
  if profile:
392
- first_name = profile.get("first_name", "")
393
- last_name = profile.get("last_name", "")
394
- title = (first_name + " " + last_name).strip()
395
 
396
  result[canonical] = (title or "VK пост", views)
397
 
398
  return result
399
 
400
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
401
  def normalize_links(links: Union[List[str], str]) -> List[str]:
402
  if isinstance(links, str):
403
  raw = links.splitlines()
@@ -411,35 +522,78 @@ def normalize_links(links: Union[List[str], str]) -> List[str]:
411
  def _escape(text: str) -> str:
412
  return html_lib.escape(text, quote=True)
413
 
414
- def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str, int, int, List[str]]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
415
  errors: List[str] = []
416
 
417
- tg_groups = {}
418
- vk_groups = {}
419
-
420
- telegram_lines = [line for line in lines if line[0] == "telegram"]
421
- vk_lines = [line for line in lines if line[0] == "vk"]
422
- telegram_results = {}
423
- vk_results = {}
424
-
425
- if telegram_lines:
426
- valid_telegram_batch: List[Tuple[str, str, str]] = []
427
- for _, canonical, username, mid in telegram_lines:
428
- if username and mid:
429
- valid_telegram_batch.append((canonical, username, mid))
430
- if valid_telegram_batch:
431
- telegram_results = process_telegram_batch(valid_telegram_batch)
432
-
433
- valid_vk_batch: List[Tuple[str, int, str]] = []
434
- for _, canonical, owner_id, post_id in vk_lines:
435
- if owner_id is not None and post_id is not None:
436
- valid_vk_batch.append((canonical, owner_id, post_id))
437
-
438
- if valid_vk_batch:
439
- batch_size = max(1, VK_BATCH_SIZE)
440
- for start in range(0, len(valid_vk_batch), batch_size):
441
- chunk = valid_vk_batch[start:start + batch_size]
442
- vk_results.update(process_vk_batch(chunk))
443
 
444
  for plat, canonical, a, b in lines:
445
  if plat == "telegram":
@@ -447,16 +601,16 @@ def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str
447
  if not username or not mid:
448
  key = canonical
449
  invalid_title = "Ошибка (TG): ссылка Telegram недоступна для публичного парсинга"
450
- if re.search(r"https?://(?:t(?:elegram)?\\.me)/c/\\d+/\\d+", canonical, re.IGNORECASE):
451
  invalid_title = "Ошибка (TG): ссылки вида t.me/c/... не поддерживаются"
452
  tg_groups.setdefault(key, {"title": invalid_title, "items": []})
453
  tg_groups[key]["items"].append((canonical, 0))
454
  else:
455
- title, views = telegram_results.get(canonical, (username, 0))
456
  key = username
457
  if key not in tg_groups:
458
  tg_groups[key] = {"title": title, "items": []}
459
- if not tg_groups[key].get("title") or str(tg_groups[key]["title"]).startswith("**Ошибка"):
460
  tg_groups[key]["title"] = title
461
  tg_groups[key]["items"].append((canonical, views))
462
 
@@ -467,29 +621,35 @@ def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str
467
  vk_groups.setdefault(key, {"title": "VK пост", "items": []})
468
  vk_groups[key]["items"].append((canonical, 0))
469
  else:
470
- title, views = vk_results.get(canonical, process_vk_link(owner_id, post_id, canonical))
471
  key = str(owner_id)
472
  if key not in vk_groups:
473
  vk_groups[key] = {"title": title, "items": []}
474
  if not vk_groups[key].get("title") or str(vk_groups[key]["title"]).startswith("**Ошибка"):
475
  vk_groups[key]["title"] = title
476
  vk_groups[key]["items"].append((canonical, views))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
477
  else:
478
  errors.append(f"Неизвестная платформа: {canonical}")
479
 
480
- tg_total_views = sum(v for g in tg_groups.values() for _, v in g["items"])
481
- vk_total_views = sum(v for g in vk_groups.values() for _, v in g["items"])
482
-
483
- tg_sorted = sorted(
484
- tg_groups.items(),
485
- key=lambda kv: sum(v for _, v in kv[1]["items"]),
486
- reverse=True,
487
- )
488
- vk_sorted = sorted(
489
- vk_groups.items(),
490
- key=lambda kv: sum(v for _, v in kv[1]["items"]),
491
- reverse=True,
492
- )
493
 
494
  html_lines: List[str] = []
495
  text_lines: List[str] = []
@@ -499,30 +659,10 @@ def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str
499
  html_lines.append(f'Суммарно посты собрали <b>{human_format_views(tg_total_views)}</b> просмотров.')
500
  text_lines.append("Telegram")
501
  text_lines.append(f"Суммарно посты собрали {human_format_views(tg_total_views)} просмотров.")
502
- if tg_sorted:
503
- html_lines.append("<ol>")
504
- for idx, (_, data) in enumerate(tg_sorted, start=1):
505
- title = data["title"] or "Telegram"
506
- items = data["items"]
507
- first_link, _first_views = items[0]
508
-
509
- title_html = _escape(title)
510
- first_link_html = _escape(first_link)
511
- line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
512
- if len(items) > 1:
513
- for link2, _v in items[1:]:
514
- line_html += f' + <a href="{_escape(link2)}">ещё</a>'
515
- views_str = " + ".join(human_format_views(v) for _, v in items)
516
- line_html += f" — {views_str}</li>"
517
- html_lines.append(line_html)
518
-
519
- line_text = f"{idx}. {title} ({first_link})"
520
- if len(items) > 1:
521
- extra_links = ", ".join(link2 for link2, _v in items[1:])
522
- line_text += f" + ещё: {extra_links}"
523
- line_text += f" — {views_str}"
524
- text_lines.append(line_text)
525
- html_lines.append("</ol>")
526
  else:
527
  html_lines.append("<i>Нет ссылок на Telegram</i>")
528
  text_lines.append("Нет ссылок на Telegram")
@@ -534,36 +674,28 @@ def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str
534
  html_lines.append(f'Суммарно посты собрали <b>{human_format_views(vk_total_views)}</b> просмотров.')
535
  text_lines.append("ВКонтакте")
536
  text_lines.append(f"Суммарно посты собрали {human_format_views(vk_total_views)} просмотров.")
537
- if vk_sorted:
538
- html_lines.append("<ol>")
539
- for idx, (_, data) in enumerate(vk_sorted, start=1):
540
- title = data["title"] or "VK пост"
541
- items = data["items"]
542
- first_link, _first_views = items[0]
543
-
544
- title_html = _escape(title)
545
- first_link_html = _escape(first_link)
546
- line_html = f'<li><a href="{first_link_html}">{title_html}</a>'
547
- if len(items) > 1:
548
- for link2, _v in items[1:]:
549
- line_html += f' + <a href="{_escape(link2)}">ещё</a>'
550
- views_str = " + ".join(human_format_views(v) for _, v in items)
551
- line_html += f" — {views_str}</li>"
552
- html_lines.append(line_html)
553
-
554
- line_text = f"{idx}. {title} ({first_link})"
555
- if len(items) > 1:
556
- extra_links = ", ".join(link2 for link2, _v in items[1:])
557
- line_text += f" + ещё: {extra_links}"
558
- line_text += f" — {views_str}"
559
- text_lines.append(line_text)
560
- html_lines.append("</ol>")
561
  else:
562
  html_lines.append("<i>Нет ссылок на ВКонтакте</i>")
563
  text_lines.append("Нет ссылок на ВКонтакте")
564
  html_lines.append("<br/>")
565
  text_lines.append("")
566
 
 
 
 
 
 
 
 
 
 
 
 
 
567
  if errors:
568
  html_lines.append("<h2>Ошибки</h2>")
569
  html_lines.append("<ul>")
@@ -574,7 +706,14 @@ def build_output(lines: List[Tuple[str, str, object, object]]) -> Tuple[str, str
574
  html_lines.append("<br/>")
575
  text_lines.append("")
576
 
577
- return "\n".join(html_lines), "\n".join(text_lines), tg_total_views, vk_total_views, errors
 
 
 
 
 
 
 
578
 
579
 
580
  @app.get("/health")
@@ -584,10 +723,17 @@ def health_check():
584
 
585
  @app.post("/parse", response_model=ParseResponse)
586
  def parse_links(payload: ParseRequest):
 
 
587
  raw_lines = normalize_links(payload.links)
588
  if not raw_lines:
589
  return ParseResponse(html="", text="", telegram_total=0, vk_total=0, errors=["Нет ссылок для обработки."])
590
 
 
 
 
 
 
591
  seen = set()
592
  lines: List[Tuple[str, str, object, object]] = []
593
  for link in raw_lines:
@@ -595,9 +741,7 @@ def parse_links(payload: ParseRequest):
595
  if plat == "telegram":
596
  canonical, username, mid = canonicalize_tg_link(link)
597
  if not canonical:
598
- canonical = link
599
- username = None
600
- mid = None
601
  if canonical in seen:
602
  continue
603
  seen.add(canonical)
@@ -610,14 +754,29 @@ def parse_links(payload: ParseRequest):
610
  continue
611
  seen.add(canonical)
612
  lines.append(("vk", canonical, owner_id, post_id))
 
 
 
 
 
 
 
 
613
  else:
614
  lines.append(("unknown", link, None, None))
615
 
616
- html, text, tg_total, vk_total, errors = build_output(lines)
 
 
 
 
 
 
617
  return ParseResponse(
618
  html=html,
619
  text=text,
620
  telegram_total=tg_total,
621
  vk_total=vk_total,
622
  errors=errors,
 
623
  )
 
1
  from __future__ import annotations
2
 
3
+ from concurrent.futures import ThreadPoolExecutor, wait
4
+ import threading
 
5
  import time
6
  import os
7
  import re
8
  import html as html_lib
9
+ from typing import List, Optional, Tuple, Union
10
 
11
  import requests
12
  from fastapi import FastAPI
 
20
  TelegramClient = None
21
  StringSession = None
22
 
 
 
 
 
 
 
23
  APP_NAME = "pr-tool-backend"
24
 
25
  VK_API_VERSION = os.getenv("VK_API_VERSION", "5.131")
 
27
  TELEGRAM_API_ID = int(os.getenv("TELEGRAM_API_ID", "0") or "0")
28
  TELEGRAM_API_HASH = os.getenv("TELEGRAM_API_HASH", "")
29
  TELEGRAM_STRING_SESSION = os.getenv("TELEGRAM_STRING_SESSION", "")
30
+
31
+ # --- Таймауты и параллелизм (всё настраивается через переменные окружения) ---
32
+ # Таймаут одного HTTP-запроса задаём кортежем (connect, read), чтобы зависший
33
+ # коннект не съедал всё время.
34
+ _TG_READ_TIMEOUT = float(os.getenv("TELEGRAM_OP_TIMEOUT", "8"))
35
+ TG_HTTP_TIMEOUT = (3.05, _TG_READ_TIMEOUT)
36
+ _MAX_READ_TIMEOUT = float(os.getenv("MAX_OP_TIMEOUT", "6"))
37
+ MAX_HTTP_TIMEOUT = (3.05, _MAX_READ_TIMEOUT)
38
+ REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "15")) # для VK API
39
+
40
+ TELEGRAM_CONCURRENCY = int(os.getenv("TELEGRAM_CONCURRENCY", "16"))
41
+ MAX_CONCURRENCY = int(os.getenv("MAX_CONCURRENCY", "8"))
42
+
43
  VK_MAX_RETRIES = int(os.getenv("VK_MAX_RETRIES", "5"))
44
  VK_RETRY_DELAY = float(os.getenv("VK_RETRY_DELAY", "0.45"))
45
  VK_BATCH_SIZE = int(os.getenv("VK_BATCH_SIZE", "100"))
 
 
46
 
47
+ # Сторонний прокси r.jina.ai заметно медленнее прямого запроса к t.me.
48
+ # По умолчанию выключен — включается только при ENABLE_JINA_FALLBACK=1.
49
+ ENABLE_JINA_FALLBACK = os.getenv("ENABLE_JINA_FALLBACK", "0") == "1"
50
+
51
+ # Общий бюджет времени на весь запрос. По истечении возвращаем то, что успели
52
+ # обработать, а остальное помечаем как «превышено время» — вместо того, чтобы
53
+ # мини-приложение отвалилось по таймауту целиком.
54
+ TOTAL_TIME_BUDGET = float(os.getenv("TOTAL_TIME_BUDGET", "25"))
55
+
56
+ # Предохранитель от слишком больших списков. Лишнее отрезаем с пометкой.
57
+ MAX_INPUT_LINKS = int(os.getenv("MAX_INPUT_LINKS", "1200"))
58
 
59
  app = FastAPI(title=APP_NAME)
60
 
 
64
  allow_origins=["*"],
65
  allow_credentials=False,
66
  allow_methods=["POST", "GET", "OPTIONS"],
67
+ allow_headers=["*"],
68
  )
69
 
70
 
71
+ # ---------- HTTP-сессии с переиспользованием соединений ----------
72
+ # Отдельная сессия на поток (потокобезопасно) + пул соединений, чтобы не
73
+ # открывать TCP/TLS заново на каждую ссылку.
74
+ _thread_local = threading.local()
75
+
76
+
77
+ def _get_session() -> requests.Session:
78
+ sess = getattr(_thread_local, "session", None)
79
+ if sess is None:
80
+ sess = requests.Session()
81
+ adapter = requests.adapters.HTTPAdapter(
82
+ pool_connections=4, pool_maxsize=4, max_retries=0
83
+ )
84
+ sess.mount("http://", adapter)
85
+ sess.mount("https://", adapter)
86
+ sess.headers.update({"User-Agent": "Mozilla/5.0"})
87
+ _thread_local.session = sess
88
+ return sess
89
+
90
+
91
  class ParseRequest(BaseModel):
92
  links: Union[List[str], str]
93
 
 
98
  telegram_total: int
99
  vk_total: int
100
  errors: List[str]
101
+ max_count: int = 0 # число ссылок MAX (просмотры из MAX недоступны)
102
 
103
 
104
  # ---------- Вспомогательные функции ----------
 
121
  title = re.sub(r"\s+", " ", title).strip()
122
  return title or fallback
123
 
124
+
125
+ def clean_max_title(raw: str, fallback: str) -> str:
126
+ title = re.sub(r"<[^>]+>", "", (raw or "").strip())
127
+ title = html_lib.unescape(title)
128
+ # убираем хвосты вида " — MAX", " | MAX", " - MAX"
129
+ title = re.sub(r"\s*[-—|·]\s*MAX\s*$", "", title, flags=re.IGNORECASE)
130
+ title = re.sub(r"\s+", " ", title).strip()
131
+ return title or fallback
132
+
133
+
134
  def human_format_views(num: int) -> str:
135
  if num >= 1_000_000:
136
  value = num / 1_000_000
 
148
  return formatted.replace(".", ",")
149
 
150
 
151
+ def _parse_views_number(raw: str) -> int:
152
+ if not raw:
153
+ return 0
154
+ normalized = raw.strip().replace("\u00a0", "").replace(" ", "").replace(",", ".")
155
+ match = re.match(r"([\d\.]+)\s*([kKmM]?)", normalized)
156
+ if not match:
157
+ return 0
158
+ value = float(match.group(1))
159
+ suffix = match.group(2).lower()
160
+ if suffix == "k":
161
+ value *= 1_000
162
+ elif suffix == "m":
163
+ value *= 1_000_000
164
+ return int(value)
165
+
166
+
167
  def detect_platform(link: str) -> str:
168
  link = link.strip().lower()
169
  if "t.me/" in link or "telegram.me/" in link:
170
  return "telegram"
171
  if "vk.com/wall" in link or "vk.ru/wall" in link:
172
  return "vk"
173
+ if "max.ru/" in link:
174
+ return "max"
175
  return ""
176
 
177
 
178
+ def canonicalize_tg_link(link: str) -> Tuple[Optional[str], Optional[str], Optional[str]]:
179
  link = link.strip()
180
  private_match = re.match(r"https?://(?:t(?:elegram)?\.me)/c/(\d+)/(\d+)", link, re.IGNORECASE)
181
  if private_match:
 
189
  return canonical, username, message_id
190
 
191
 
192
+ def canonicalize_vk_link(link: str) -> Tuple[Optional[str], Optional[int], Optional[str]]:
193
  link = link.strip()
194
  m = re.search(r"(?:vk\.com|vk\.ru)/wall(-?\d+)_(\d+)", link)
195
  if not m:
 
200
  return canonical, owner_id, post_id
201
 
202
 
203
+ def canonicalize_max_link(link: str) -> Tuple[Optional[str], Optional[str], Optional[str]]:
204
+ link = link.strip()
205
+ m = re.match(r"https?://(?:[\w-]+\.)?max\.ru/([^/?#]+)/([^/?#]+)", link, re.IGNORECASE)
206
+ if not m:
207
+ return None, None, None
208
+ slug = m.group(1)
209
+ post_id = m.group(2)
210
+ canonical = f"https://max.ru/{slug}/{post_id}"
211
+ return canonical, slug, post_id
212
+
213
+
214
  def _telethon_ready() -> bool:
215
  return bool(
216
  TelegramClient
 
221
  )
222
 
223
 
224
+ # ---------- Обработчики одной ссылки ----------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
225
 
226
  def process_telegram_link(channel_username: str, message_id: str, canonical_link: str) -> Tuple[str, int]:
227
+ primary_urls = [
228
  f"https://t.me/{channel_username}/{message_id}?embed=1",
229
  f"https://t.me/s/{channel_username}/{message_id}",
 
 
 
 
230
  ]
231
+ fallback_urls = []
232
+ if ENABLE_JINA_FALLBACK:
233
+ fallback_urls = [
234
+ f"https://r.jina.ai/https://t.me/{channel_username}/{message_id}?embed=1",
235
+ ]
236
+
237
+ session = _get_session()
238
+ page_html = None
239
+ last_err: Optional[Exception] = None
240
+ for url in primary_urls + fallback_urls:
241
+ try:
242
+ resp = session.get(url, timeout=TG_HTTP_TIMEOUT)
243
+ if resp.status_code == 200 and resp.text:
244
+ page_html = resp.text
245
+ break
246
+ except Exception as exc: # noqa: BLE001
247
+ last_err = exc
 
 
 
 
 
 
 
 
 
 
248
 
249
+ if page_html is None:
250
+ return f"Ошибка (TG) при обработке {canonical_link}: {last_err or 'нет ответа'}", 0
251
 
252
+ title = None
253
+ meta_title = re.search(
254
+ r"<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
255
+ page_html,
256
+ re.IGNORECASE,
257
+ )
258
+ if meta_title:
259
+ title = meta_title.group(1).strip()
260
+ if not title:
261
+ owner_title = re.search(
262
+ r'class="tgme_widget_message_owner_name".*?<span[^>]*>(.*?)</span>',
263
  page_html,
264
+ re.IGNORECASE | re.DOTALL,
265
  )
266
+ if owner_title:
267
+ title = owner_title.group(1).strip()
268
+ title = clean_telegram_title(title or "", channel_username)
269
+
270
+ views = 0
271
+ widget_views = re.search(
272
+ r'class="tgme_widget_message_views[^"]*">([\d\s\.,kKmM]+)<',
273
+ page_html,
274
+ re.IGNORECASE,
275
+ )
276
+ if widget_views:
277
+ views = _parse_views_number(widget_views.group(1))
278
+ else:
279
+ candidates = re.findall(
280
+ r"(\d[\d\s\.,]*)([kKmM]?)\s*(?:views|просмотр|просмотра|просмотров|переглядів|visualizações|visualizzazioni|ansichten|visninger|visitas)",
281
  page_html,
282
  re.IGNORECASE,
283
  )
284
+ if candidates:
285
+ last_num, last_suffix = candidates[-1]
286
+ views = _parse_views_number(last_num + last_suffix)
287
+
288
+ return title, views
289
+
290
+
291
+ def process_max_link(slug: str, post_id: str, canonical_link: str) -> Tuple[str, None]:
292
+ """MAX: подхватываем название канала из страницы поста.
293
+ Просмотры из MAX недоступны, поэтому всегда возвращаем None."""
294
+ title = slug
295
+ try:
296
+ session = _get_session()
297
+ resp = session.get(canonical_link, timeout=MAX_HTTP_TIMEOUT)
298
+ if resp.status_code == 200 and resp.text:
299
+ page_html = resp.text
300
+ m = re.search(
301
+ r"<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']",
302
  page_html,
303
  re.IGNORECASE,
304
  )
305
+ if not m:
306
+ m = re.search(r"<title[^>]*>(.*?)</title>", page_html, re.IGNORECASE | re.DOTALL)
307
+ if m:
308
+ title = clean_max_title(m.group(1), slug)
309
+ except Exception: # noqa: BLE001
310
+ title = slug # тихо откатываемся к слагу из URL
311
+ return title or slug, None
312
 
313
 
314
  def process_vk_link(owner_id: int, post_id: str, canonical_link: str) -> Tuple[str, int]:
315
+ """Одиночный запрос к VK (запасной путь; основной — батч process_vk_batch)."""
316
  posts_param = f"{owner_id}_{post_id}"
317
  api_url = "https://api.vk.com/method/wall.getById"
318
 
319
+ data = None
320
  for attempt in range(1, VK_MAX_RETRIES + 1):
321
+ params = {"posts": posts_param, "v": VK_API_VERSION, "extended": 1}
 
 
 
 
322
  if VK_ACCESS_TOKEN:
323
  params["access_token"] = VK_ACCESS_TOKEN
 
324
  try:
325
  resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
326
+ except Exception as exc: # noqa: BLE001
327
  return f"**Ошибка (VK)**: {exc} — {canonical_link}", 0
 
328
  if resp.status_code != 200:
329
  return f"**Ошибка (VK)**: HTTP {resp.status_code} — {canonical_link}", 0
 
330
  data = resp.json()
331
  if "error" in data:
332
  error = data["error"]
 
340
  else:
341
  return f"**Ошибка (VK)**: Too many requests per second — {canonical_link}", 0
342
 
343
+ response_data = (data or {}).get("response", {})
344
  items = response_data.get("items", [])
345
  if not items:
346
  return f"**Ошибка (VK)**: пост не найден — {canonical_link}", 0
 
357
  title = group.get("name")
358
  break
359
  else:
 
360
  for profile in response_data.get("profiles", []):
361
+ if profile.get("id") == post_owner_id:
362
+ title = (profile.get("first_name", "") + " " + profile.get("last_name", "")).strip()
 
 
363
  break
364
  if not title:
365
  title = "VK пост"
 
366
  return title, views
367
 
368
 
369
+ def process_vk_batch(batch_items: List[Tuple[str, int, str]]) -> dict:
 
 
370
  api_url = "https://api.vk.com/method/wall.getById"
371
  posts = ",".join(f"{owner_id}_{post_id}" for _, owner_id, post_id in batch_items)
372
 
373
+ data = None
374
  for attempt in range(1, VK_MAX_RETRIES + 1):
375
+ params = {"posts": posts, "v": VK_API_VERSION, "extended": 1}
 
 
 
 
376
  if VK_ACCESS_TOKEN:
377
  params["access_token"] = VK_ACCESS_TOKEN
 
378
  try:
379
  resp = requests.get(api_url, params=params, timeout=REQUEST_TIMEOUT)
380
+ except Exception as exc: # noqa: BLE001
381
+ return {c: (f"**Ошибка (VK)**: {exc} — {c}", 0) for c, _, _ in batch_items}
 
 
 
382
 
383
  if resp.status_code != 200:
384
+ return {c: (f"**Ошибка (VK)**: HTTP {resp.status_code} — {c}", 0) for c, _, _ in batch_items}
 
 
 
385
 
386
  data = resp.json()
387
  if "error" in data:
 
391
  if error_code == 6 or "too many requests per second" in error_msg.lower():
392
  time.sleep(VK_RETRY_DELAY * attempt)
393
  continue
394
+ return {c: (f"**Ошибка (VK)**: {error_msg or 'API error'} — {c}", 0) for c, _, _ in batch_items}
 
 
 
395
  break
396
  else:
397
+ return {c: (f"**Ошибка (VK)**: Too many requests per second — {c}", 0) for c, _, _ in batch_items}
 
 
 
398
 
399
+ response_data = (data or {}).get("response", {})
400
  items = response_data.get("items", [])
401
  groups = {group.get("id"): group for group in response_data.get("groups", [])}
402
  profiles = {profile.get("id"): profile for profile in response_data.get("profiles", [])}
 
406
  if item.get("owner_id") is not None and item.get("id") is not None
407
  }
408
 
409
+ result: dict = {}
410
  for canonical, owner_id, post_id in batch_items:
411
  key = f"{owner_id}_{post_id}"
412
  post = item_map.get(key)
 
417
  views = post.get("views", {}).get("count", 0)
418
  post_owner_id = post.get("owner_id", owner_id)
419
  title = None
 
420
  if post_owner_id < 0:
421
  group = groups.get(-post_owner_id)
422
  if group:
 
424
  else:
425
  profile = profiles.get(post_owner_id)
426
  if profile:
427
+ title = (profile.get("first_name", "") + " " + profile.get("last_name", "")).strip()
 
 
428
 
429
  result[canonical] = (title or "VK пост", views)
430
 
431
  return result
432
 
433
 
434
+ # ---------- Параллельное разрешение ссылок с бюджетом времени ----------
435
+
436
+ def _resolve_pool(tasks, worker, max_workers, deadline, timeout_label, defaults=None):
437
+ """tasks: список (canonical, args_tuple). worker(*args) -> (title, views).
438
+ Возвращаем dict canonical -> (title, views). Незавершённые к дедлайну
439
+ помечаем timeout_label (или оставляем значение из defaults, если оно есть)."""
440
+ results = dict(defaults) if defaults else {}
441
+ if not tasks:
442
+ return results
443
+
444
+ workers = max(1, min(max_workers, len(tasks)))
445
+ ex = ThreadPoolExecutor(max_workers=workers)
446
+ fut_map = {ex.submit(worker, *args): canonical for canonical, args in tasks}
447
+
448
+ timeout = max(0.0, deadline - time.monotonic())
449
+ done, not_done = wait(list(fut_map.keys()), timeout=timeout)
450
+
451
+ for fut in done:
452
+ canonical = fut_map[fut]
453
+ try:
454
+ results[canonical] = fut.result()
455
+ except Exception as exc: # noqa: BLE001
456
+ if canonical not in results:
457
+ results[canonical] = (f"Ошибка при обработке {canonical}: {exc}", 0)
458
+
459
+ for fut in not_done:
460
+ canonical = fut_map[fut]
461
+ fut.cancel()
462
+ if canonical not in results:
463
+ results[canonical] = (timeout_label, 0)
464
+
465
+ # Не блокируемся на «зависших» запросах — отдаём управление сразу.
466
+ try:
467
+ ex.shutdown(wait=False, cancel_futures=True)
468
+ except TypeError: # Python < 3.9
469
+ ex.shutdown(wait=False)
470
+
471
+ return results
472
+
473
+
474
+ def resolve_telegram(tg_tasks, deadline):
475
+ # tg_tasks: список (canonical, username, message_id)
476
+ tasks = [(c, (u, mid, c)) for (c, u, mid) in tg_tasks if u and mid]
477
+ return _resolve_pool(
478
+ tasks,
479
+ process_telegram_link,
480
+ TELEGRAM_CONCURRENCY,
481
+ deadline,
482
+ "Ошибка (TG): превышено время обработки",
483
+ )
484
+
485
+
486
+ def resolve_max(max_tasks, deadline):
487
+ # max_tasks: список (canonical, slug, post_id)
488
+ defaults = {c: (slug, None) for (c, slug, _post) in max_tasks}
489
+ tasks = [(c, (slug, post_id, c)) for (c, slug, post_id) in max_tasks]
490
+ fetched = _resolve_pool(
491
+ tasks, process_max_link, MAX_CONCURRENCY, deadline, None, defaults=defaults
492
+ )
493
+ # просмотры из MAX недоступны всегда
494
+ return {c: (t, None) for c, (t, _v) in fetched.items()}
495
+
496
+
497
+ def resolve_vk(vk_tasks, deadline):
498
+ # vk_tasks: список (canonical, owner_id, post_id)
499
+ results = {}
500
+ valid = [(c, o, p) for (c, o, p) in vk_tasks if o is not None and p is not None]
501
+ batch_size = max(1, VK_BATCH_SIZE)
502
+ for start in range(0, len(valid), batch_size):
503
+ if time.monotonic() > deadline:
504
+ for c, _, _ in valid[start:]:
505
+ results[c] = ("**Ошибка (VK)**: превышено время обработки", 0)
506
+ break
507
+ chunk = valid[start:start + batch_size]
508
+ results.update(process_vk_batch(chunk))
509
+ return results
510
+
511
+
512
  def normalize_links(links: Union[List[str], str]) -> List[str]:
513
  if isinstance(links, str):
514
  raw = links.splitlines()
 
522
  def _escape(text: str) -> str:
523
  return html_lib.escape(text, quote=True)
524
 
525
+
526
+ def _fmt_views(v) -> str:
527
+ return "—" if v is None else human_format_views(v)
528
+
529
+
530
+ def _render_groups(groups, has_views, default_title):
531
+ """Сортируем группы и рендерим в HTML (<ol>) и текст. Возвращаем (html_str, text_lines)."""
532
+ if has_views:
533
+ ordered = sorted(
534
+ groups.items(),
535
+ key=lambda kv: sum((v or 0) for _, v in kv[1]["items"]),
536
+ reverse=True,
537
+ )
538
+ else:
539
+ ordered = sorted(groups.items(), key=lambda kv: (kv[1]["title"] or "").lower())
540
+
541
+ html_lines = ["<ol>"]
542
+ text_lines: List[str] = []
543
+ for idx, (_, data) in enumerate(ordered, start=1):
544
+ title = data["title"] or default_title
545
+ items = data["items"]
546
+ first_link, _first_views = items[0]
547
+
548
+ line_html = f'<li><a href="{_escape(first_link)}">{_escape(title)}</a>'
549
+ if len(items) > 1:
550
+ for link2, _v in items[1:]:
551
+ line_html += f' + <a href="{_escape(link2)}">ещё</a>'
552
+
553
+ if has_views:
554
+ views_str = " + ".join(_fmt_views(v) for _, v in items)
555
+ line_html += f" — {views_str}</li>"
556
+ else:
557
+ line_html += " — </li>"
558
+ html_lines.append(line_html)
559
+
560
+ line_text = f"{idx}. {title} ({first_link})"
561
+ if len(items) > 1:
562
+ line_text += " + ещё: " + ", ".join(link2 for link2, _v in items[1:])
563
+ if has_views:
564
+ line_text += f" — {views_str}"
565
+ else:
566
+ line_text += " —"
567
+ text_lines.append(line_text)
568
+
569
+ html_lines.append("</ol>")
570
+ return "\n".join(html_lines), text_lines
571
+
572
+
573
+ def build_output(lines, deadline):
574
  errors: List[str] = []
575
 
576
+ tg_lines = [ln for ln in lines if ln[0] == "telegram"]
577
+ vk_lines = [ln for ln in lines if ln[0] == "vk"]
578
+ max_lines = [ln for ln in lines if ln[0] == "max"]
579
+
580
+ tg_fetch = [(c, a, b) for _, c, a, b in tg_lines if a and b]
581
+ vk_fetch = [(c, a, b) for _, c, a, b in vk_lines if a is not None and b is not None]
582
+ max_fetch = [(c, a, b) for _, c, a, b in max_lines if a and b]
583
+
584
+ # Telegram, VK и MAX обрабатываем параллельно (раньше шли друг за другом).
585
+ tg_results, vk_results, max_results = {}, {}, {}
586
+ with ThreadPoolExecutor(max_workers=3) as top:
587
+ f_tg = top.submit(resolve_telegram, tg_fetch, deadline)
588
+ f_vk = top.submit(resolve_vk, vk_fetch, deadline)
589
+ f_max = top.submit(resolve_max, max_fetch, deadline)
590
+ tg_results = f_tg.result()
591
+ vk_results = f_vk.result()
592
+ max_results = f_max.result()
593
+
594
+ tg_groups: dict = {}
595
+ vk_groups: dict = {}
596
+ max_groups: dict = {}
 
 
 
 
 
597
 
598
  for plat, canonical, a, b in lines:
599
  if plat == "telegram":
 
601
  if not username or not mid:
602
  key = canonical
603
  invalid_title = "Ошибка (TG): ссылка Telegram недоступна для публичного парсинга"
604
+ if re.search(r"https?://(?:t(?:elegram)?\.me)/c/\d+/\d+", canonical, re.IGNORECASE):
605
  invalid_title = "Ошибка (TG): ссылки вида t.me/c/... не поддерживаются"
606
  tg_groups.setdefault(key, {"title": invalid_title, "items": []})
607
  tg_groups[key]["items"].append((canonical, 0))
608
  else:
609
+ title, views = tg_results.get(canonical, (username, 0))
610
  key = username
611
  if key not in tg_groups:
612
  tg_groups[key] = {"title": title, "items": []}
613
+ if not tg_groups[key].get("title") or str(tg_groups[key]["title"]).startswith("Ошибка"):
614
  tg_groups[key]["title"] = title
615
  tg_groups[key]["items"].append((canonical, views))
616
 
 
621
  vk_groups.setdefault(key, {"title": "VK пост", "items": []})
622
  vk_groups[key]["items"].append((canonical, 0))
623
  else:
624
+ title, views = vk_results.get(canonical, (f"**Ошибка (VK)**: нет данных — {canonical}", 0))
625
  key = str(owner_id)
626
  if key not in vk_groups:
627
  vk_groups[key] = {"title": title, "items": []}
628
  if not vk_groups[key].get("title") or str(vk_groups[key]["title"]).startswith("**Ошибка"):
629
  vk_groups[key]["title"] = title
630
  vk_groups[key]["items"].append((canonical, views))
631
+
632
+ elif plat == "max":
633
+ slug, post_id = a, b
634
+ if not slug or not post_id:
635
+ key = canonical
636
+ max_groups.setdefault(key, {"title": "MAX", "items": []})
637
+ max_groups[key]["items"].append((canonical, None))
638
+ else:
639
+ title, _views = max_results.get(canonical, (slug, None))
640
+ key = slug
641
+ if key not in max_groups:
642
+ max_groups[key] = {"title": title, "items": []}
643
+ if not max_groups[key].get("title"):
644
+ max_groups[key]["title"] = title
645
+ max_groups[key]["items"].append((canonical, None))
646
+
647
  else:
648
  errors.append(f"Неизвестная платформа: {canonical}")
649
 
650
+ tg_total_views = sum(v for g in tg_groups.values() for _, v in g["items"] if v)
651
+ vk_total_views = sum(v for g in vk_groups.values() for _, v in g["items"] if v)
652
+ max_count = sum(len(g["items"]) for g in max_groups.values())
 
 
 
 
 
 
 
 
 
 
653
 
654
  html_lines: List[str] = []
655
  text_lines: List[str] = []
 
659
  html_lines.append(f'Суммарно посты собрали <b>{human_format_views(tg_total_views)}</b> просмотров.')
660
  text_lines.append("Telegram")
661
  text_lines.append(f"Суммарно посты собрали {human_format_views(tg_total_views)} просмотров.")
662
+ if tg_groups:
663
+ h, t = _render_groups(tg_groups, has_views=True, default_title="Telegram")
664
+ html_lines.append(h)
665
+ text_lines.extend(t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
666
  else:
667
  html_lines.append("<i>Нет ссылок на Telegram</i>")
668
  text_lines.append("Нет ссылок на Telegram")
 
674
  html_lines.append(f'Суммарно посты собрали <b>{human_format_views(vk_total_views)}</b> просмотров.')
675
  text_lines.append("ВКонтакте")
676
  text_lines.append(f"Суммарно посты собрали {human_format_views(vk_total_views)} просмотров.")
677
+ if vk_groups:
678
+ h, t = _render_groups(vk_groups, has_views=True, default_title="VK пост")
679
+ html_lines.append(h)
680
+ text_lines.extend(t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
681
  else:
682
  html_lines.append("<i>Нет ссылок на ВКонтакте</i>")
683
  text_lines.append("Нет ссылок на ВКонтакте")
684
  html_lines.append("<br/>")
685
  text_lines.append("")
686
 
687
+ # --- MAX (секцию показываем только если есть ссылки MAX) ---
688
+ if max_groups:
689
+ html_lines.append("<h2>MAX</h2>")
690
+ html_lines.append("<i>Просмотры из MAX недоступны.</i>")
691
+ text_lines.append("MAX")
692
+ text_lines.append("Просмотры из MAX недоступны.")
693
+ h, t = _render_groups(max_groups, has_views=False, default_title="MAX")
694
+ html_lines.append(h)
695
+ text_lines.extend(t)
696
+ html_lines.append("<br/>")
697
+ text_lines.append("")
698
+
699
  if errors:
700
  html_lines.append("<h2>Ошибки</h2>")
701
  html_lines.append("<ul>")
 
706
  html_lines.append("<br/>")
707
  text_lines.append("")
708
 
709
+ return (
710
+ "\n".join(html_lines),
711
+ "\n".join(text_lines),
712
+ tg_total_views,
713
+ vk_total_views,
714
+ errors,
715
+ max_count,
716
+ )
717
 
718
 
719
  @app.get("/health")
 
723
 
724
  @app.post("/parse", response_model=ParseResponse)
725
  def parse_links(payload: ParseRequest):
726
+ deadline = time.monotonic() + TOTAL_TIME_BUDGET
727
+
728
  raw_lines = normalize_links(payload.links)
729
  if not raw_lines:
730
  return ParseResponse(html="", text="", telegram_total=0, vk_total=0, errors=["Нет ссылок для обработки."])
731
 
732
+ truncated = False
733
+ if len(raw_lines) > MAX_INPUT_LINKS:
734
+ raw_lines = raw_lines[:MAX_INPUT_LINKS]
735
+ truncated = True
736
+
737
  seen = set()
738
  lines: List[Tuple[str, str, object, object]] = []
739
  for link in raw_lines:
 
741
  if plat == "telegram":
742
  canonical, username, mid = canonicalize_tg_link(link)
743
  if not canonical:
744
+ canonical, username, mid = link, None, None
 
 
745
  if canonical in seen:
746
  continue
747
  seen.add(canonical)
 
754
  continue
755
  seen.add(canonical)
756
  lines.append(("vk", canonical, owner_id, post_id))
757
+ elif plat == "max":
758
+ canonical, slug, post_id = canonicalize_max_link(link)
759
+ if not canonical:
760
+ canonical, slug, post_id = link, None, None
761
+ if canonical in seen:
762
+ continue
763
+ seen.add(canonical)
764
+ lines.append(("max", canonical, slug, post_id))
765
  else:
766
  lines.append(("unknown", link, None, None))
767
 
768
+ html, text, tg_total, vk_total, errors, max_count = build_output(lines, deadline)
769
+
770
+ if truncated:
771
+ note = f"Обработаны первые {MAX_INPUT_LINKS} ссылок (список был длиннее)."
772
+ errors = errors + [note]
773
+ text = text + ("\n" if text else "") + "- " + note
774
+
775
  return ParseResponse(
776
  html=html,
777
  text=text,
778
  telegram_total=tg_total,
779
  vk_total=vk_total,
780
  errors=errors,
781
+ max_count=max_count,
782
  )