Spaces:
Running
Running
Upload receiver.py
Browse files- receiver.py +106 -7
receiver.py
CHANGED
|
@@ -11,6 +11,8 @@ import random
|
|
| 11 |
import re
|
| 12 |
import secrets
|
| 13 |
import signal
|
|
|
|
|
|
|
| 14 |
import sqlite3
|
| 15 |
import threading
|
| 16 |
import time
|
|
@@ -26,6 +28,7 @@ from urllib.parse import urlsplit
|
|
| 26 |
from zoneinfo import ZoneInfo
|
| 27 |
|
| 28 |
LOG = logging.getLogger("monitor")
|
|
|
|
| 29 |
MAX_BYTES = 1024 * 1024
|
| 30 |
AI_CHUNK = 12000
|
| 31 |
DEFAULT_PROMPT = """Ты классификатор публикаций для мониторинга нового внешнего вида
|
|
@@ -280,28 +283,121 @@ class ServiceError(Exception):
|
|
| 280 |
self.code, self.retry_after = code, retry_after
|
| 281 |
|
| 282 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
def request_json(url, payload, headers=None, service="api", timeout=45):
|
| 284 |
request = urllib.request.Request(url, data=json.dumps(payload, ensure_ascii=False).encode(),
|
| 285 |
headers={"Content-Type": "application/json", **(headers or {})}, method="POST")
|
| 286 |
try:
|
| 287 |
with urllib.request.urlopen(request, timeout=timeout) as response:
|
| 288 |
-
|
|
|
|
|
|
|
|
|
|
| 289 |
except urllib.error.HTTPError as exc:
|
| 290 |
try:
|
| 291 |
details = json.loads(exc.read(MAX_BYTES))
|
| 292 |
except (ValueError, OSError):
|
| 293 |
details = {}
|
|
|
|
|
|
|
| 294 |
if service == "telegram" and "message is not modified" in str(details.get("description", "")).lower():
|
| 295 |
return {"ok": True, "result": True}
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
|
| 301 |
|
| 302 |
class Clients:
|
| 303 |
def __init__(self, config):
|
| 304 |
self.config, self.last_tg = config, 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
|
| 306 |
def classify(self, post, index):
|
| 307 |
text = post["text"][index:index + AI_CHUNK]
|
|
@@ -420,7 +516,7 @@ class Monitor:
|
|
| 420 |
thread.start()
|
| 421 |
|
| 422 |
def status(self):
|
| 423 |
-
return {"mode": "monitor", "missing": self.config.missing(), "model": self.config.model,
|
| 424 |
"storage": "postgresql" if self.config.database_url else "temporary_sqlite",
|
| 425 |
"queue": self.store.stats(), "workers": {t.name: t.is_alive() for t in self.threads}}
|
| 426 |
|
|
@@ -454,7 +550,9 @@ def make_server(monitor, host="0.0.0.0", port=7860):
|
|
| 454 |
if urlsplit(self.path).path in ("/", "/health"):
|
| 455 |
missing = monitor.config.missing()
|
| 456 |
return self.reply(200, {"status": "configuration_required" if missing else "monitor_ready",
|
| 457 |
-
"missing": missing, "storage": "postgresql" if monitor.config.database_url else "temporary_sqlite"})
|
|
|
|
|
|
|
| 458 |
if self.authorized("/status/") or self.authorized("/samples/"):
|
| 459 |
try:
|
| 460 |
return self.reply(200, monitor.status())
|
|
@@ -513,3 +611,4 @@ if __name__ == "__main__":
|
|
| 513 |
finally:
|
| 514 |
app.stop.set()
|
| 515 |
server.server_close()
|
|
|
|
|
|
| 11 |
import re
|
| 12 |
import secrets
|
| 13 |
import signal
|
| 14 |
+
import socket
|
| 15 |
+
import ssl
|
| 16 |
import sqlite3
|
| 17 |
import threading
|
| 18 |
import time
|
|
|
|
| 28 |
from zoneinfo import ZoneInfo
|
| 29 |
|
| 30 |
LOG = logging.getLogger("monitor")
|
| 31 |
+
VERSION = "1.1-diagnostics"
|
| 32 |
MAX_BYTES = 1024 * 1024
|
| 33 |
AI_CHUNK = 12000
|
| 34 |
DEFAULT_PROMPT = """Ты классификатор публикаций для мониторинга нового внешнего вида
|
|
|
|
| 283 |
self.code, self.retry_after = code, retry_after
|
| 284 |
|
| 285 |
|
| 286 |
+
def telegram_error_category(details):
|
| 287 |
+
"""Only fixed labels: never expose arbitrary remote text or secret URLs."""
|
| 288 |
+
description = str(details.get("description", "")).lower()
|
| 289 |
+
for phrase, label in (
|
| 290 |
+
("unauthorized", "unauthorized"), ("chat not found", "chat_not_found"),
|
| 291 |
+
("bot was kicked", "bot_kicked"), ("bot is not a member", "bot_not_member"),
|
| 292 |
+
("bot was blocked", "bot_blocked"), ("not enough rights", "insufficient_rights"),
|
| 293 |
+
("have no rights", "insufficient_rights"), ("chat_write_forbidden", "write_forbidden"),
|
| 294 |
+
("can't parse entities", "invalid_html"), ("message is too long", "message_too_long"),
|
| 295 |
+
("message thread not found", "thread_not_found"),
|
| 296 |
+
("group chat was upgraded", "chat_migrated"),
|
| 297 |
+
("too many requests", "rate_limited"),
|
| 298 |
+
):
|
| 299 |
+
if phrase in description:
|
| 300 |
+
return label
|
| 301 |
+
return "api_rejected"
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def network_error_category(exc):
|
| 305 |
+
reason = exc.reason if isinstance(exc, urllib.error.URLError) else exc
|
| 306 |
+
if isinstance(reason, socket.gaierror):
|
| 307 |
+
return "dns_error"
|
| 308 |
+
if isinstance(reason, (TimeoutError, socket.timeout)):
|
| 309 |
+
return "timeout"
|
| 310 |
+
if isinstance(reason, ssl.SSLError):
|
| 311 |
+
return "tls_error"
|
| 312 |
+
if isinstance(reason, ConnectionRefusedError):
|
| 313 |
+
return "connection_refused"
|
| 314 |
+
if isinstance(reason, ConnectionResetError):
|
| 315 |
+
return "connection_reset"
|
| 316 |
+
return "network_error"
|
| 317 |
+
|
| 318 |
+
|
| 319 |
def request_json(url, payload, headers=None, service="api", timeout=45):
|
| 320 |
request = urllib.request.Request(url, data=json.dumps(payload, ensure_ascii=False).encode(),
|
| 321 |
headers={"Content-Type": "application/json", **(headers or {})}, method="POST")
|
| 322 |
try:
|
| 323 |
with urllib.request.urlopen(request, timeout=timeout) as response:
|
| 324 |
+
result = json.loads(response.read(2 * MAX_BYTES))
|
| 325 |
+
if not isinstance(result, dict):
|
| 326 |
+
raise ServiceError(service + "_invalid_json_object")
|
| 327 |
+
return result
|
| 328 |
except urllib.error.HTTPError as exc:
|
| 329 |
try:
|
| 330 |
details = json.loads(exc.read(MAX_BYTES))
|
| 331 |
except (ValueError, OSError):
|
| 332 |
details = {}
|
| 333 |
+
if not isinstance(details, dict):
|
| 334 |
+
details = {}
|
| 335 |
if service == "telegram" and "message is not modified" in str(details.get("description", "")).lower():
|
| 336 |
return {"ok": True, "result": True}
|
| 337 |
+
parameters = details.get("parameters") or {}
|
| 338 |
+
parameters = parameters if isinstance(parameters, dict) else {}
|
| 339 |
+
delay = parameters.get("retry_after", 0) if service == "telegram" else 0
|
| 340 |
+
try:
|
| 341 |
+
delay = float(delay or 0)
|
| 342 |
+
except (ValueError, TypeError):
|
| 343 |
+
delay = 0
|
| 344 |
+
suffix = "_" + telegram_error_category(details) if service == "telegram" else ""
|
| 345 |
+
raise ServiceError(f"{service}_http_{exc.code}" + suffix, delay) from None
|
| 346 |
+
except ValueError:
|
| 347 |
+
raise ServiceError(service + "_invalid_json") from None
|
| 348 |
+
except OSError as exc:
|
| 349 |
+
raise ServiceError(service + "_" + network_error_category(exc)) from None
|
| 350 |
|
| 351 |
|
| 352 |
class Clients:
|
| 353 |
def __init__(self, config):
|
| 354 |
self.config, self.last_tg = config, 0.0
|
| 355 |
+
self.diagnostics_lock = threading.Lock()
|
| 356 |
+
|
| 357 |
+
def diagnose_telegram(self):
|
| 358 |
+
"""Read-only Bot API checks; no publication, queue changes or AI calls."""
|
| 359 |
+
if not self.diagnostics_lock.acquire(blocking=False):
|
| 360 |
+
return {"version": VERSION, "status": "diagnostics_busy"}
|
| 361 |
+
report = {"version": VERSION, "checks": []}
|
| 362 |
+
try:
|
| 363 |
+
if not self.config.telegram_token or not self.config.chat_id:
|
| 364 |
+
return {"version": VERSION, "status": "telegram_configuration_missing"}
|
| 365 |
+
bot_id = None
|
| 366 |
+
for method in ("getMe", "getChat", "getChatMember"):
|
| 367 |
+
payload = {} if method == "getMe" else {"chat_id": self.config.chat_id}
|
| 368 |
+
if method == "getChatMember":
|
| 369 |
+
payload["user_id"] = bot_id
|
| 370 |
+
started = time.monotonic()
|
| 371 |
+
try:
|
| 372 |
+
result = request_json(
|
| 373 |
+
"https://api.telegram.org/bot" + self.config.telegram_token + "/" + method,
|
| 374 |
+
payload, service="telegram", timeout=10)
|
| 375 |
+
if not result.get("ok") or not isinstance(result.get("result"), dict):
|
| 376 |
+
raise ServiceError("telegram_invalid_response")
|
| 377 |
+
data = result["result"]
|
| 378 |
+
check = {"method": method, "ok": True}
|
| 379 |
+
if method == "getMe":
|
| 380 |
+
bot_id = data.get("id")
|
| 381 |
+
if not isinstance(bot_id, int):
|
| 382 |
+
raise ServiceError("telegram_invalid_bot_id")
|
| 383 |
+
check["bot_username"] = data.get("username")
|
| 384 |
+
elif method == "getChat":
|
| 385 |
+
check.update(chat_id=data.get("id"), chat_type=data.get("type"),
|
| 386 |
+
permissions=data.get("permissions", {}))
|
| 387 |
+
else:
|
| 388 |
+
check["membership"] = data.get("status")
|
| 389 |
+
for field in ("is_member", "can_send_messages", "can_post_messages", "until_date"):
|
| 390 |
+
if field in data:
|
| 391 |
+
check[field] = data[field]
|
| 392 |
+
except ServiceError as exc:
|
| 393 |
+
check = {"method": method, "ok": False, "error": exc.code}
|
| 394 |
+
check["seconds"] = round(time.monotonic() - started, 2)
|
| 395 |
+
report["checks"].append(check)
|
| 396 |
+
if not check["ok"]:
|
| 397 |
+
break
|
| 398 |
+
return report
|
| 399 |
+
finally:
|
| 400 |
+
self.diagnostics_lock.release()
|
| 401 |
|
| 402 |
def classify(self, post, index):
|
| 403 |
text = post["text"][index:index + AI_CHUNK]
|
|
|
|
| 516 |
thread.start()
|
| 517 |
|
| 518 |
def status(self):
|
| 519 |
+
return {"mode": "monitor", "version": VERSION, "missing": self.config.missing(), "model": self.config.model,
|
| 520 |
"storage": "postgresql" if self.config.database_url else "temporary_sqlite",
|
| 521 |
"queue": self.store.stats(), "workers": {t.name: t.is_alive() for t in self.threads}}
|
| 522 |
|
|
|
|
| 550 |
if urlsplit(self.path).path in ("/", "/health"):
|
| 551 |
missing = monitor.config.missing()
|
| 552 |
return self.reply(200, {"status": "configuration_required" if missing else "monitor_ready",
|
| 553 |
+
"version": VERSION, "missing": missing, "storage": "postgresql" if monitor.config.database_url else "temporary_sqlite"})
|
| 554 |
+
if self.authorized("/diagnostics/"):
|
| 555 |
+
return self.reply(200, monitor.clients.diagnose_telegram())
|
| 556 |
if self.authorized("/status/") or self.authorized("/samples/"):
|
| 557 |
try:
|
| 558 |
return self.reply(200, monitor.status())
|
|
|
|
| 611 |
finally:
|
| 612 |
app.stop.set()
|
| 613 |
server.server_close()
|
| 614 |
+
|