Cxops commited on
Commit
b07113d
·
verified ·
1 Parent(s): fcee356

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +202 -275
app.py CHANGED
@@ -1,9 +1,20 @@
1
- import asyncio, base64, hashlib, json, logging, os, re, secrets, shutil, signal, sqlite3, sys, tempfile, threading, time, uuid, unicodedata
 
 
 
 
 
 
 
 
 
 
 
2
  from pathlib import Path
3
  from typing import Optional, Tuple, Union
4
- from datetime import datetime, timezone
5
 
6
- import aiohttp
 
7
  from flask import Flask
8
  from pyrogram import Client, filters, errors as pyro_errors
9
  from pyrogram.types import InlineKeyboardMarkup, InlineKeyboardButton, Message, CallbackQuery
@@ -11,7 +22,7 @@ from huggingface_hub import HfApi, hf_hub_download, hf_hub_url, utils as hf_util
11
  from cryptography.hazmat.primitives.ciphers.aead import AESSIV
12
 
13
  # ---------------------------------------------------------------------------
14
- # Environment validation
15
  # ---------------------------------------------------------------------------
16
  REQUIRED_ENV = ["API_ID", "API_HASH", "BOT_TOKEN", "HF_TOKEN"]
17
  for var in REQUIRED_ENV:
@@ -27,25 +38,26 @@ SPACE_HOST = os.environ.get("SPACE_HOST", "http://localhost:7860")
27
 
28
  DATASET_REPO = "Cxops/Bzxv"
29
  DATASET_TYPE = "dataset"
30
- MAX_SINGLE_FILE = 45 * 1024**3 # 45 GB
31
- MIN_FREE_DISK = 45 * 1024**3 # 45 GB
32
- DATASET_MAX = int(8.7 * 1024**4) # 8.7 TB
33
- QUOTA_PER_USER = 500 * 1024**3 # 500 GB
34
  KEEPALIVE_INTERVAL = 300
35
  UPLOAD_TIMEOUT = 7200
36
- DOWNLOAD_TIMEOUT = 3600
37
  MAX_WORKERS = 3
38
  RATE_LIMIT_UPLOADS = 5
39
  RATE_LIMIT_WINDOW = 600
 
40
 
41
- # Logging
42
- logging.basicConfig(level=logging.INFO,
43
- format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
44
- handlers=[logging.StreamHandler(sys.stdout)])
 
45
  logger = logging.getLogger("StorageBot")
46
 
47
  # ---------------------------------------------------------------------------
48
- # Encryption (AES‑SIV)
49
  # ---------------------------------------------------------------------------
50
  _siv_key = hashlib.sha512(OBFUSCATION_KEY.encode()).digest()
51
  _aes_siv = AESSIV(_siv_key)
@@ -71,93 +83,84 @@ def enc_filename(orig: str) -> str:
71
  return encrypt_text(orig)
72
 
73
  # ---------------------------------------------------------------------------
74
- # Database Setup
75
  # ---------------------------------------------------------------------------
76
- DB_PATH = "bot.db"
77
- _db = sqlite3.connect(DB_PATH, check_same_thread=False)
78
- _db.row_factory = sqlite3.Row
79
- _db.execute("PRAGMA journal_mode=WAL")
80
- _db.execute("PRAGMA busy_timeout=5000")
81
-
82
- def _migrate():
83
- # Schema version via user_version
84
- ver = _db.execute("PRAGMA user_version").fetchone()[0]
 
 
 
 
 
 
 
85
  if ver < 1:
86
- _db.executescript("""
87
- CREATE TABLE IF NOT EXISTS files (
88
- user_id INTEGER NOT NULL,
89
- enc_path TEXT UNIQUE NOT NULL,
90
- orig_name TEXT NOT NULL,
91
- size INTEGER NOT NULL CHECK(size >= 0),
92
- timestamp REAL NOT NULL DEFAULT (strftime('%s','now'))
93
- );
94
- CREATE INDEX IF NOT EXISTS idx_files_user ON files(user_id);
95
- CREATE INDEX IF NOT EXISTS idx_files_name ON files(orig_name);
96
-
97
- CREATE TABLE IF NOT EXISTS job_queue (
98
- job_id TEXT PRIMARY KEY NOT NULL,
99
- user_id INTEGER NOT NULL,
100
- chat_id INTEGER NOT NULL,
101
- file_id TEXT NOT NULL,
102
- orig_filename TEXT NOT NULL,
103
- expected_size INTEGER,
104
- status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','processing','completed','failed','cancelled')),
105
- created REAL NOT NULL DEFAULT (strftime('%s','now')),
106
- updated REAL NOT NULL DEFAULT (strftime('%s','now'))
107
- );
108
- CREATE INDEX IF NOT EXISTS idx_jobs_status ON job_queue(status);
109
-
110
- CREATE TABLE IF NOT EXISTS callback_tokens (
111
- token TEXT PRIMARY KEY NOT NULL,
112
- type TEXT NOT NULL,
113
- data TEXT NOT NULL, -- JSON
114
- created REAL NOT NULL DEFAULT (strftime('%s','now'))
115
- );
116
-
117
- CREATE TABLE IF NOT EXISTS user_quota (
118
- user_id INTEGER PRIMARY KEY NOT NULL,
119
- used_bytes INTEGER NOT NULL DEFAULT 0 CHECK(used_bytes >= 0)
120
- );
121
- PRAGMA user_version = 1;
122
  """)
123
- # future migrations here
124
- _db.commit()
125
-
126
- _migrate()
127
 
128
  # ---------------------------------------------------------------------------
129
- # Async DB Helpers (all DB access serialised via a single async lock)
130
  # ---------------------------------------------------------------------------
131
- _db_lock = asyncio.Lock()
132
-
133
  async def db_execute(sql: str, params: tuple = ()) -> None:
134
- def _run():
135
- with sqlite3.connect(DB_PATH) as conn:
136
- conn.row_factory = sqlite3.Row
137
- conn.execute("PRAGMA journal_mode=WAL")
138
- conn.execute(sql, params)
139
- conn.commit()
140
- async with _db_lock:
141
- await asyncio.to_thread(_run)
142
-
143
- async def db_fetchone(sql: str, params: tuple = ()) -> Optional[sqlite3.Row]:
144
- def _run():
145
- with sqlite3.connect(DB_PATH) as conn:
146
- conn.row_factory = sqlite3.Row
147
- return conn.execute(sql, params).fetchone()
148
- async with _db_lock:
149
- return await asyncio.to_thread(_run)
150
-
151
- async def db_fetchall(sql: str, params: tuple = ()) -> list[sqlite3.Row]:
152
- def _run():
153
- with sqlite3.connect(DB_PATH) as conn:
154
- conn.row_factory = sqlite3.Row
155
- return conn.execute(sql, params).fetchall()
156
- async with _db_lock:
157
- return await asyncio.to_thread(_run)
158
 
159
  # ---------------------------------------------------------------------------
160
- # Rate Limiter (in‑memory)
161
  # ---------------------------------------------------------------------------
162
  class RateLimiter:
163
  def __init__(self):
@@ -170,7 +173,6 @@ class RateLimiter:
170
  if user_id not in self._users:
171
  self._users[user_id] = []
172
  timestamps = self._users[user_id]
173
- # Remove old entries
174
  self._users[user_id] = [t for t in timestamps if now - t < RATE_LIMIT_WINDOW]
175
  if len(self._users[user_id]) >= RATE_LIMIT_UPLOADS:
176
  return False
@@ -192,21 +194,18 @@ def get_free_disk() -> int:
192
  return shutil.disk_usage("/").free
193
 
194
  def total_memory() -> int:
195
- # cgroup v1
196
  try:
197
  with open("/sys/fs/cgroup/memory/memory.limit_in_bytes") as f:
198
  val = int(f.read().strip())
199
- if val < 9_223_372_036_854_771_200: # less than unlimited
200
  return val
201
  except: pass
202
- # cgroup v2
203
  try:
204
  with open("/sys/fs/cgroup/memory.max") as f:
205
  val = f.read().strip()
206
  if val != "max":
207
  return int(val)
208
  except: pass
209
- # /proc/meminfo
210
  try:
211
  with open("/proc/meminfo") as f:
212
  for line in f:
@@ -217,10 +216,8 @@ def total_memory() -> int:
217
 
218
  def sanitize_filename(name: str) -> str:
219
  name = unicodedata.normalize('NFC', name)
220
- # remove control chars and problematic symbols
221
  name = re.sub(r'[\x00-\x1f\x7f-\x9f]', '', name)
222
  name = re.sub(r'[\\/*?:"<>|]', '_', name)
223
- # Trim to 200 chars preserving extension
224
  if len(name) > 200:
225
  stem, ext = os.path.splitext(name)
226
  stem = stem[:195 - len(ext)] + "..."
@@ -228,27 +225,28 @@ def sanitize_filename(name: str) -> str:
228
  return name or "unnamed"
229
 
230
  # ---------------------------------------------------------------------------
231
- # FloodWait retry wrapper
232
  # ---------------------------------------------------------------------------
233
- async def flood_retry(callable, *args, **kwargs):
234
- """Retry a callable (sync or async) with exponential backoff on FloodWait."""
 
 
 
235
  for attempt in range(5):
236
  try:
237
- if asyncio.iscoroutinefunction(callable):
238
- return await callable(*args, **kwargs)
239
- else:
240
- return await asyncio.to_thread(callable, *args, **kwargs)
241
  except pyro_errors.FloodWait as e:
242
  wait = e.value + 1
243
  logger.warning(f"FloodWait {wait}s")
244
  await asyncio.sleep(wait)
245
  except pyro_errors.RPCError as e:
246
- if attempt == 4: raise
 
247
  await asyncio.sleep(2 ** attempt)
248
  return None
249
 
250
  # ---------------------------------------------------------------------------
251
- # Flask App with Health Endpoints
252
  # ---------------------------------------------------------------------------
253
  web = Flask(__name__)
254
 
@@ -268,7 +266,7 @@ def run_web():
268
  web.run(host="0.0.0.0", port=7860, use_reloader=False)
269
 
270
  # ---------------------------------------------------------------------------
271
- # Hugging Face API Wrappers (async via to_thread)
272
  # ---------------------------------------------------------------------------
273
  hf = HfApi(token=HF_TOKEN)
274
 
@@ -300,18 +298,11 @@ async def hf_list_files():
300
  return await asyncio.to_thread(hf.list_repo_files, repo_id=DATASET_REPO, repo_type=DATASET_TYPE)
301
 
302
  # ---------------------------------------------------------------------------
303
- # Quota Management
304
  # ---------------------------------------------------------------------------
305
- async def _quota_lock(lock: asyncio.Lock = asyncio.Lock()):
306
- """Serialise quota updates."""
307
- async with lock:
308
- yield
309
- # We'll use a per-user lock? simpler: a global quota lock for all quota ops.
310
-
311
  _quota_lock = asyncio.Lock()
312
 
313
  async def reserve_quota(user_id: int, bytes_: int) -> bool:
314
- """Atomically check and update reservation. Returns True if within quota."""
315
  async with _quota_lock:
316
  row = await db_fetchone("SELECT used_bytes FROM user_quota WHERE user_id=?", (user_id,))
317
  used = row["used_bytes"] if row else 0
@@ -330,10 +321,8 @@ async def release_quota(user_id: int, bytes_: int):
330
  (bytes_, user_id)
331
  )
332
 
333
- async def reconcile_quota_on_startup():
334
- """Rebuild user_quota from files table."""
335
  async with _quota_lock:
336
- # reset
337
  await db_execute("DELETE FROM user_quota")
338
  rows = await db_fetchall("SELECT user_id, COALESCE(SUM(size),0) AS total FROM files GROUP BY user_id")
339
  for row in rows:
@@ -341,7 +330,7 @@ async def reconcile_quota_on_startup():
341
  (row["user_id"], row["total"]))
342
 
343
  # ---------------------------------------------------------------------------
344
- # Job Queue (with proper state transitions)
345
  # ---------------------------------------------------------------------------
346
  async def enqueue_job(user_id, chat_id, file_id, orig_filename, expected_size):
347
  job_id = secrets.token_hex(16)
@@ -352,37 +341,29 @@ async def enqueue_job(user_id, chat_id, file_id, orig_filename, expected_size):
352
  return job_id
353
 
354
  async def dequeue_job() -> Optional[Tuple[str, int, int, str, str, int]]:
355
- """Atomic dequeue of a pending job."""
356
- async with _db_lock:
357
- def _run():
358
- with sqlite3.connect(DB_PATH) as conn:
359
- conn.row_factory = sqlite3.Row
360
- conn.execute("PRAGMA journal_mode=WAL")
361
- conn.execute("BEGIN IMMEDIATE")
362
- row = conn.execute(
363
- "SELECT job_id, user_id, chat_id, file_id, orig_filename, expected_size FROM job_queue WHERE status='pending' ORDER BY created LIMIT 1"
364
- ).fetchone()
365
- if row:
366
- conn.execute("UPDATE job_queue SET status='processing', updated=strftime('%s','now') WHERE job_id=?",
367
- (row["job_id"],))
368
- conn.commit()
369
- return row
370
- return await asyncio.to_thread(_run)
371
 
372
  async def complete_job(job_id: str):
373
  await db_execute("DELETE FROM job_queue WHERE job_id=?", (job_id,))
374
 
375
- async def fail_job(job_id: str):
376
- await db_execute("UPDATE job_queue SET status='failed', updated=strftime('%s','now') WHERE job_id=?", (job_id,))
377
-
378
  async def cancel_job(job_id: str):
379
- await db_execute("UPDATE job_queue SET status='cancelled', updated=strftime('%s','now') WHERE job_id=? AND status IN ('pending','processing')", (job_id,))
380
 
381
  async def reset_processing_jobs():
382
  await db_execute("UPDATE job_queue SET status='pending' WHERE status='processing'")
383
 
384
  # ---------------------------------------------------------------------------
385
- # Token Storage (JSON payloads)
386
  # ---------------------------------------------------------------------------
387
  async def store_token(token: str, type_: str, data: dict):
388
  await db_execute("INSERT OR REPLACE INTO callback_tokens (token, type, data, created) VALUES (?,?,?,?)",
@@ -390,7 +371,7 @@ async def store_token(token: str, type_: str, data: dict):
390
 
391
  async def get_token_data(token: str) -> Optional[dict]:
392
  row = await db_fetchone("SELECT data FROM callback_tokens WHERE token=? AND created > ?",
393
- (token, time.time() - 3600)) # 1 hour TTL
394
  if row:
395
  return json.loads(row["data"])
396
  return None
@@ -399,15 +380,16 @@ async def delete_token(token: str):
399
  await db_execute("DELETE FROM callback_tokens WHERE token=?", (token,))
400
 
401
  # ---------------------------------------------------------------------------
402
- # File listing helper (reusable, with pagination)
403
  # ---------------------------------------------------------------------------
404
  async def build_file_list_message(user_id: int, page: int = 0, per_page: int = 10) -> Tuple[str, Optional[InlineKeyboardMarkup]]:
405
- rows = await db_fetchall("SELECT enc_path, orig_name, size FROM files WHERE user_id=? ORDER BY timestamp DESC LIMIT ? OFFSET ?",
406
- (user_id, per_page + 1, page * per_page))
 
 
407
  has_more = len(rows) > per_page
408
  if has_more:
409
  rows = rows[:per_page]
410
-
411
  if not rows and page == 0:
412
  return "You have no uploaded files.", None
413
 
@@ -420,42 +402,37 @@ async def build_file_list_message(user_id: int, page: int = 0, per_page: int = 1
420
  InlineKeyboardButton("🗑 Delete", callback_data=f"delete:{token}")
421
  ])
422
 
423
- nav_buttons = []
424
  if page > 0:
425
- nav_buttons.append(InlineKeyboardButton("⬅️ Prev", callback_data=f"files_page:{page-1}"))
426
  if has_more:
427
- nav_buttons.append(InlineKeyboardButton("Next ➡️", callback_data=f"files_page:{page+1}"))
428
- if nav_buttons:
429
- buttons.append(nav_buttons)
430
 
431
  buttons.append([InlineKeyboardButton("⚠️ Delete All Files", callback_data="delete_all_confirm")])
432
  return "Your uploaded files:", InlineKeyboardMarkup(buttons)
433
 
434
  # ---------------------------------------------------------------------------
435
- # Upload Worker (multi‑worker, cancellation support)
436
  # ---------------------------------------------------------------------------
437
  active_uploads: dict[str, asyncio.Task] = {}
438
 
439
  async def process_upload(job_id: str, user_id: int, chat_id: int, file_id: str, orig_filename: str, expected_size: int):
440
- # Check disk space
441
  needed = (expected_size or 0) + MIN_FREE_DISK
442
- while True:
443
- if get_free_disk() >= needed:
444
- break
445
  logger.warning("Low disk, waiting...")
446
  await asyncio.sleep(30)
447
 
448
- # Rate limit
449
  if not await rate_limiter.is_allowed(user_id):
450
  await flood_retry(app.send_message, chat_id, "⚠️ Too many uploads, try later.")
451
  return
452
 
453
- # Check dataset capacity
454
  try:
455
  info = await hf_dataset_info()
456
  current_used = info.used_storage or 0
457
  if current_used + (expected_size or 0) > DATASET_MAX:
458
- await flood_retry(app.send_message, chat_id, "❌ Dataset is full. Upload rejected.")
459
  return
460
  except Exception as e:
461
  logger.warning(f"Could not fetch dataset info: {e}")
@@ -475,7 +452,6 @@ async def process_upload(job_id: str, user_id: int, chat_id: int, file_id: str,
475
  await msg.edit(f"❌ File too large ({format_size(file_size)}).")
476
  return
477
 
478
- # Quota check
479
  if not await reserve_quota(user_id, file_size):
480
  await msg.edit(f"❌ Quota exceeded ({format_size(QUOTA_PER_USER)}).")
481
  return
@@ -484,7 +460,6 @@ async def process_upload(job_id: str, user_id: int, chat_id: int, file_id: str,
484
  repo_path = f"{enc_folder(user_id)}/{enc_filename(safe_name)}"
485
  await msg.edit(f"⬆️ Uploading `{safe_name}` to Hugging Face...")
486
 
487
- # Retry upload with exponential backoff
488
  for attempt in range(3):
489
  try:
490
  await hf_upload_file(local_path, repo_path)
@@ -492,14 +467,10 @@ async def process_upload(job_id: str, user_id: int, chat_id: int, file_id: str,
492
  except hf_utils.HfHubHTTPError as e:
493
  if attempt == 2:
494
  raise
495
- wait = 2 ** attempt
496
- logger.warning(f"HF upload attempt {attempt+1} failed, retrying in {wait}s")
497
- await asyncio.sleep(wait)
498
 
499
- # Verify upload
500
  meta = await hf_get_metadata(repo_path)
501
  if meta.size != file_size:
502
- logger.error(f"Size mismatch local={file_size} remote={meta.size}")
503
  await hf_delete_file(repo_path)
504
  raise RuntimeError("Upload verification failed")
505
 
@@ -509,7 +480,6 @@ async def process_upload(job_id: str, user_id: int, chat_id: int, file_id: str,
509
  logger.info(f"Uploaded {orig_filename} for user {user_id}")
510
 
511
  except asyncio.CancelledError:
512
- # Cancellation: cleanup partial upload
513
  logger.info(f"Job {job_id} cancelled")
514
  if local_path and os.path.exists(local_path):
515
  os.remove(local_path)
@@ -535,9 +505,7 @@ async def upload_worker(worker_id: int):
535
  continue
536
  job_id, user_id, chat_id, file_id, orig_filename, expected_size = job
537
  logger.info(f"Worker {worker_id} processing {orig_filename}")
538
- task = asyncio.create_task(
539
- process_upload(job_id, user_id, chat_id, file_id, orig_filename, expected_size)
540
- )
541
  active_uploads[job_id] = task
542
  try:
543
  await task
@@ -548,7 +516,7 @@ async def upload_worker(worker_id: int):
548
  await complete_job(job_id)
549
 
550
  # ---------------------------------------------------------------------------
551
- # Download helper (splitting large files)
552
  # ---------------------------------------------------------------------------
553
  async def do_download(chat_id: int, enc_path: str, orig_name: str):
554
  temp_dir = tempfile.mkdtemp()
@@ -590,7 +558,7 @@ app = Client("storage_bot", api_id=API_ID, api_hash=API_HASH, bot_token=BOT_TOKE
590
 
591
  @app.on_message(filters.command("start"))
592
  async def start_cmd(_, msg: Message):
593
- await msg.reply("👋 Welcome! Send a file or a t.me link.\n/files – list files\n/status – stats")
594
 
595
  @app.on_message(filters.command("help"))
596
  async def help_cmd(_, msg: Message):
@@ -619,10 +587,9 @@ async def status_cmd(_, msg: Message):
619
  @app.on_message(filters.command("files"))
620
  async def files_cmd(_, msg: Message):
621
  user_id = msg.from_user.id
622
- # Reconcile if empty
623
  if not await db_fetchone("SELECT 1 FROM files WHERE user_id=?", (user_id,)):
624
  await reconcile_user(user_id)
625
- text, markup = await build_file_list_message(user_id, page=0)
626
  if markup is None:
627
  await msg.reply(text)
628
  else:
@@ -633,7 +600,8 @@ async def reconcile_user(user_id: int):
633
  try:
634
  hf_files = await hf_list_files()
635
  hf_paths = [f for f in hf_files if f.startswith(enc_f + "/")]
636
- db_paths = [r[0] for r in await db_fetchall("SELECT enc_path FROM files WHERE user_id=?", (user_id,))]
 
637
  for p in db_paths:
638
  if p not in hf_paths:
639
  await db_execute("DELETE FROM files WHERE enc_path=?", (p,))
@@ -655,11 +623,10 @@ async def reconcile_user(user_id: int):
655
 
656
  @app.on_message(filters.command("delete_all"))
657
  async def delete_all_cmd(_, msg: Message):
658
- user_id = msg.from_user.id
659
  kb = InlineKeyboardMarkup([[
660
  InlineKeyboardButton("Yes, delete all", callback_data="delete_all_confirm")
661
  ]])
662
- await msg.reply("⚠️ This will permanently delete all your files. Confirm?", reply_markup=kb)
663
 
664
  @app.on_message(filters.command("upload"))
665
  async def upload_cmd(_, msg: Message):
@@ -667,51 +634,47 @@ async def upload_cmd(_, msg: Message):
667
  if len(args) < 2:
668
  return await msg.reply("Usage: /upload <link>")
669
  link = args[1].strip()
670
- match = re.search(r'https?://t\.me/((?:c/)?[\w\d_]+)/(\d+)', link)
671
  if not match:
672
  return await msg.reply("Invalid link.")
673
- chat_part, msg_id = match.group(1), int(match.group(2))
674
- if chat_part.startswith("c/"):
675
- chat_id = int("-100" + chat_part[2:])
676
- else:
677
- chat_id = chat_part
678
  try:
679
- target_msg = await app.get_messages(chat_id, msg_id)
 
 
 
 
680
  except Exception as e:
681
- return await msg.reply(f"❌ Could not fetch message: {e}")
682
- if not target_msg.media:
683
  return await msg.reply("No file in that message.")
684
- file_id, size, name = _extract_media(target_msg)
685
- if not name:
686
- return await msg.reply("Unsupported media.")
687
  if size and size > MAX_SINGLE_FILE:
688
  return await msg.reply(f"❌ File too large ({format_size(size)})")
689
- await handle_upload_request(msg.from_user.id, msg.chat.id, file_id, name, size)
690
 
691
  def _extract_media(msg: Message):
692
  if msg.document:
693
  return msg.document.file_id, msg.document.file_size, msg.document.file_name
694
- elif msg.video:
695
  return msg.video.file_id, msg.video.file_size, msg.video.file_name or f"{msg.video.file_unique_id}.mp4"
696
- elif msg.audio:
697
  return msg.audio.file_id, msg.audio.file_size, msg.audio.file_name or f"{msg.audio.file_unique_id}.mp3"
698
- elif msg.photo:
699
  return msg.photo.file_id, msg.photo.file_size, f"{msg.photo.file_unique_id}.jpg"
700
- elif msg.animation:
701
  return msg.animation.file_id, msg.animation.file_size, msg.animation.file_name or f"{msg.animation.file_unique_id}.mp4"
702
- elif msg.sticker:
703
- if msg.sticker.is_animated: ext = ".tgs"
704
- elif msg.sticker.is_video: ext = ".webm"
705
- else: ext = ".webp"
706
  return msg.sticker.file_id, msg.sticker.file_size, f"{msg.sticker.file_unique_id}{ext}"
707
- elif msg.video_note:
708
  return msg.video_note.file_id, msg.video_note.file_size, f"{msg.video_note.file_unique_id}.mp4"
709
- elif msg.voice:
710
  return msg.voice.file_id, msg.voice.file_size, f"{msg.voice.file_unique_id}.ogg"
711
  return None, None, None
712
 
713
  async def handle_upload_request(user_id, chat_id, file_id, filename, size):
714
- # Check duplicate by file_unique_id? We'll just do filename+size for simplicity.
715
  existing = await db_fetchone("SELECT enc_path FROM files WHERE user_id=? AND orig_name=? AND size=?",
716
  (user_id, filename, size))
717
  if existing:
@@ -730,15 +693,13 @@ async def handle_upload_request(user_id, chat_id, file_id, filename, size):
730
  job_id = await enqueue_job(user_id, chat_id, file_id, filename, size)
731
  await flood_retry(app.send_message, chat_id, f"📦 `{filename}` queued.")
732
 
733
- # ---------------------------------------------------------------------------
734
  # Callback handlers
735
- # ---------------------------------------------------------------------------
736
  @app.on_callback_query(filters.regex("^get:"))
737
  async def get_callback(_, query: CallbackQuery):
738
  token = query.data.split(":", 1)[1]
739
  data = await get_token_data(token)
740
  if not data or data.get("user_id") != query.from_user.id:
741
- return await query.answer("Invalid or expired.", show_alert=True)
742
  enc_path = data["enc_path"]
743
  orig_name = enc_path.split("/")[-1]
744
  try: orig_name = decrypt_text(orig_name)
@@ -751,24 +712,22 @@ async def delete_callback(_, query: CallbackQuery):
751
  token = query.data.split(":", 1)[1]
752
  data = await get_token_data(token)
753
  if not data or data.get("user_id") != query.from_user.id:
754
- return await query.answer("Invalid or expired.", show_alert=True)
755
  enc_path = data["enc_path"]
756
  try:
757
  await hf_delete_file(enc_path)
758
  except: pass
759
- # Release quota
760
  row = await db_fetchone("SELECT size FROM files WHERE enc_path=?", (enc_path,))
761
  if row:
762
  await release_quota(query.from_user.id, row["size"])
763
  await db_execute("DELETE FROM files WHERE enc_path=?", (enc_path,))
764
  await delete_token(token)
765
  await query.answer("Deleted.")
766
- # Refresh list (re‑send message)
767
- text, markup = await build_file_list_message(query.from_user.id, page=0)
768
  await query.message.edit_text(text, reply_markup=markup)
769
 
770
  @app.on_callback_query(filters.regex("^delete_all_confirm$"))
771
- async def delete_all_confirm_callback(_, query: CallbackQuery):
772
  user_id = query.from_user.id
773
  rows = await db_fetchall("SELECT enc_path, size FROM files WHERE user_id=?", (user_id,))
774
  for r in rows:
@@ -779,56 +738,45 @@ async def delete_all_confirm_callback(_, query: CallbackQuery):
779
  await query.message.edit_text("All files deleted.")
780
 
781
  @app.on_callback_query(filters.regex("^files_page:"))
782
- async def files_page_callback(_, query: CallbackQuery):
783
- page = int(query.data.split(":", 1)[1])
784
  text, markup = await build_file_list_message(query.from_user.id, page=page)
785
  await query.message.edit_text(text, reply_markup=markup)
786
  await query.answer()
787
 
788
  @app.on_callback_query(filters.regex("^dup_skip:"))
789
- async def dup_skip_callback(_, query: CallbackQuery):
790
- token = query.data.split(":", 1)[1]
791
- await delete_token(token)
792
  await query.message.edit_text("Skipped.")
793
 
794
  @app.on_callback_query(filters.regex("^dup_keep:"))
795
- async def dup_keep_callback(_, query: CallbackQuery):
796
- token = query.data.split(":", 1)[1]
797
  data = await get_token_data(token)
798
- if not data:
799
- return await query.answer("Expired.")
800
- user_id = data["user_id"]
801
- chat_id = data["chat_id"]
802
- file_id = data["file_id"]
803
- filename = data["filename"]
804
- size = data["size"]
805
- # Generate unique name: filename_<random>.ext
806
- stem, ext = os.path.splitext(filename)
807
  new_name = f"{stem}_{secrets.token_hex(3)}{ext}"
808
- job_id = await enqueue_job(user_id, chat_id, file_id, new_name, size)
809
  await query.message.edit_text(f"📦 Duplicate queued as `{new_name}`.")
810
  await delete_token(token)
811
 
812
  @app.on_callback_query(filters.regex("^cancel_job:"))
813
  async def cancel_job_callback(_, query: CallbackQuery):
814
- job_id = query.data.split(":", 1)[1]
815
  if job_id in active_uploads:
816
  active_uploads[job_id].cancel()
817
  await query.answer("Cancelling...")
818
  else:
819
- await db_execute("UPDATE job_queue SET status='cancelled' WHERE job_id=? AND status='pending'", (job_id,))
820
- await query.answer("Cancelled (if pending).")
821
 
822
- # ---------------------------------------------------------------------------
823
- # Album handling (simplified, robust)
824
- # ---------------------------------------------------------------------------
825
  _album_cache: dict[str, dict] = {}
826
  _album_lock = asyncio.Lock()
827
 
828
  @app.on_message(filters.media & filters.media_group)
829
  async def album_handler(_, msg: Message):
830
- if not msg.media_group_id:
831
- return
832
  group_id = str(msg.media_group_id)
833
  async with _album_lock:
834
  if group_id not in _album_cache:
@@ -847,7 +795,6 @@ async def album_handler(_, msg: Message):
847
  msgs = _album_cache[group_id]["msgs"]
848
  else:
849
  return
850
- # Extract all files
851
  files_info = []
852
  for m in msgs:
853
  fid, size, name = _extract_media(m)
@@ -855,15 +802,13 @@ async def album_handler(_, msg: Message):
855
  files_info.append((fid, size, name, m.id))
856
  if not files_info:
857
  return
858
- # Check for oversized
859
- oversized = [(fid,size,name) for (fid,size,name,_) in files_info if size and size > MAX_SINGLE_FILE]
860
  if oversized:
861
- names = ", ".join(n for _,_,n in oversized)
862
- await flood_retry(app.send_message, msgs[0].chat.id, f"❌ Skipping {names} – exceeds {format_size(MAX_SINGLE_FILE)}")
863
- files_info = [(fid,size,name,mid) for (fid,size,name,mid) in files_info if not (size and size > MAX_SINGLE_FILE)]
864
  if not files_info:
865
  return
866
- # Build prompt with Upload All + individual
867
  token = secrets.token_hex(8)
868
  await store_token(token, "album", {"user_id": msgs[0].from_user.id, "files": files_info})
869
  rows = []
@@ -877,12 +822,10 @@ async def album_handler(_, msg: Message):
877
  await flood_retry(app.send_message, msgs[0].chat.id,
878
  f"Album of {len(files_info)} files:\n{file_list}\nChoose:",
879
  reply_markup=InlineKeyboardMarkup(rows))
880
- # Clean album cache after processing
881
  async with _album_lock:
882
  if group_id in _album_cache:
883
  del _album_cache[group_id]
884
 
885
- # Schedule finalization
886
  async with _album_lock:
887
  _album_cache[group_id]["timer"] = asyncio.create_task(finalize())
888
 
@@ -892,7 +835,7 @@ async def upload_all_callback(_, query: CallbackQuery):
892
  data = await get_token_data(token)
893
  if not data or data.get("user_id") != query.from_user.id:
894
  return await query.answer("Expired.")
895
- for fid, size, name, mid in data["files"]:
896
  await handle_upload_request(data["user_id"], query.message.chat.id, fid, name, size)
897
  await query.message.edit_text("All files queued.")
898
  await delete_token(token)
@@ -908,29 +851,25 @@ async def upload_single_callback(_, query: CallbackQuery):
908
  await delete_token(token)
909
 
910
  @app.on_callback_query(filters.regex("^cancel_album$"))
911
- async def cancel_album_callback(_, query: CallbackQuery):
912
  await query.message.delete()
913
 
914
- # ---------------------------------------------------------------------------
915
- # Direct file upload prompt (non‑album)
916
- # ---------------------------------------------------------------------------
917
  @app.on_message(filters.document | filters.video | filters.audio | filters.photo |
918
  filters.animation | filters.sticker | filters.video_note | filters.voice)
919
- async def direct_file_handler(_, msg: Message):
920
  if msg.media_group_id:
921
- return # handled by album handler
922
  fid, size, name = _extract_media(msg)
923
  if not name: return
924
  if size and size > MAX_SINGLE_FILE:
925
  return await msg.reply(f"❌ File too large ({format_size(size)}).")
926
  await handle_upload_request(msg.from_user.id, msg.chat.id, fid, name, size)
927
 
928
- # ---------------------------------------------------------------------------
929
- # Message link handler (interactive)
930
- # ---------------------------------------------------------------------------
931
  @app.on_message(filters.text & ~filters.command(["start","help","status","files","delete_all","upload"]) &
932
  filters.regex(r'https?://t\.me/'))
933
- async def message_link_handler(_, msg: Message):
934
  match = re.search(r'(https?://t\.me/((?:c/)?[\w\d_]+)/(\d+))', msg.text)
935
  if not match: return
936
  chat_part, msg_id = match.group(2), int(match.group(3))
@@ -951,7 +890,7 @@ async def message_link_handler(_, msg: Message):
951
  await handle_upload_request(msg.from_user.id, msg.chat.id, fid, name, size)
952
 
953
  # ---------------------------------------------------------------------------
954
- # Keep‑alive (separate thread)
955
  # ---------------------------------------------------------------------------
956
  def keepalive():
957
  while True:
@@ -960,61 +899,49 @@ def keepalive():
960
  except: pass
961
  time.sleep(KEEPALIVE_INTERVAL)
962
 
963
- # ---------------------------------------------------------------------------
964
- # Main & graceful shutdown
965
- # ---------------------------------------------------------------------------
966
  bot_start_time = time.time()
967
 
968
  async def main():
969
- # Startup tasks
970
  await reset_processing_jobs()
971
- await reconcile_quota_on_startup()
972
- # Launch workers
973
  for i in range(MAX_WORKERS):
974
  asyncio.create_task(upload_worker(i))
975
- # Token cleanup task
976
  async def token_cleanup():
977
  while True:
978
  await asyncio.sleep(3600)
979
  await db_execute("DELETE FROM callback_tokens WHERE created < ?", (time.time() - 3600,))
980
  asyncio.create_task(token_cleanup())
981
- # Start Pyrogram
982
  await app.start()
983
  logger.info("Bot started")
984
- # Wait until cancelled
985
  stop_event = asyncio.Event()
986
  try:
987
  await stop_event.wait()
988
  except asyncio.CancelledError:
989
  pass
990
  finally:
991
- # Graceful shutdown
992
- logger.info("Shutting down...")
993
- # Cancel all upload workers
994
  for t in active_uploads.values():
995
  t.cancel()
996
- # Wait briefly
997
  await asyncio.sleep(1)
998
  await app.stop()
 
 
999
  logger.info("Bot stopped")
1000
 
1001
- def _shutdown_signal_handler(loop):
1002
- logger.info("Received shutdown signal")
1003
  for task in asyncio.all_tasks(loop):
1004
  task.cancel()
1005
 
1006
  if __name__ == "__main__":
1007
  loop = asyncio.new_event_loop()
1008
  asyncio.set_event_loop(loop)
1009
- # Flask in a daemon thread
1010
  threading.Thread(target=run_web, daemon=True).start()
1011
  threading.Thread(target=keepalive, daemon=True).start()
1012
- # Register signal handlers for graceful shutdown
1013
  for sig in (signal.SIGINT, signal.SIGTERM):
1014
  try:
1015
- loop.add_signal_handler(sig, lambda: _shutdown_signal_handler(loop))
1016
  except NotImplementedError:
1017
- # Windows fallback
1018
  pass
1019
  try:
1020
  loop.run_until_complete(main())
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Production‑grade Telegram ↔ Hugging Face Storage Bot.
4
+ - AES‑SIV deterministic encryption per user.
5
+ - Async SQLite via aiosqlite (no blocking I/O).
6
+ - Full media support, albums, message links.
7
+ - Persistent job queue, multi‑worker, true cancellation.
8
+ - Quota accounting, atomic operations, startup reconciliation.
9
+ - All handlers fully implemented, FloodWait safe.
10
+ """
11
+
12
+ import asyncio, base64, hashlib, json, logging, os, re, secrets, shutil, signal, sys, tempfile, threading, time, unicodedata
13
  from pathlib import Path
14
  from typing import Optional, Tuple, Union
 
15
 
16
+ import aiosqlite
17
+ import requests
18
  from flask import Flask
19
  from pyrogram import Client, filters, errors as pyro_errors
20
  from pyrogram.types import InlineKeyboardMarkup, InlineKeyboardButton, Message, CallbackQuery
 
22
  from cryptography.hazmat.primitives.ciphers.aead import AESSIV
23
 
24
  # ---------------------------------------------------------------------------
25
+ # Config
26
  # ---------------------------------------------------------------------------
27
  REQUIRED_ENV = ["API_ID", "API_HASH", "BOT_TOKEN", "HF_TOKEN"]
28
  for var in REQUIRED_ENV:
 
38
 
39
  DATASET_REPO = "Cxops/Bzxv"
40
  DATASET_TYPE = "dataset"
41
+ MAX_SINGLE_FILE = 45 * 1024**3
42
+ MIN_FREE_DISK = 45 * 1024**3
43
+ DATASET_MAX = int(8.7 * 1024**4)
44
+ QUOTA_PER_USER = 500 * 1024**3
45
  KEEPALIVE_INTERVAL = 300
46
  UPLOAD_TIMEOUT = 7200
 
47
  MAX_WORKERS = 3
48
  RATE_LIMIT_UPLOADS = 5
49
  RATE_LIMIT_WINDOW = 600
50
+ DB_PATH = "bot.db"
51
 
52
+ logging.basicConfig(
53
+ level=logging.INFO,
54
+ format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
55
+ handlers=[logging.StreamHandler(sys.stdout)],
56
+ )
57
  logger = logging.getLogger("StorageBot")
58
 
59
  # ---------------------------------------------------------------------------
60
+ # Encryption
61
  # ---------------------------------------------------------------------------
62
  _siv_key = hashlib.sha512(OBFUSCATION_KEY.encode()).digest()
63
  _aes_siv = AESSIV(_siv_key)
 
83
  return encrypt_text(orig)
84
 
85
  # ---------------------------------------------------------------------------
86
+ # Async Database (aiosqlite)
87
  # ---------------------------------------------------------------------------
88
+ _db_conn: Optional[aiosqlite.Connection] = None
89
+
90
+ async def get_db() -> aiosqlite.Connection:
91
+ global _db_conn
92
+ if _db_conn is None:
93
+ _db_conn = await aiosqlite.connect(DB_PATH)
94
+ _db_conn.row_factory = aiosqlite.Row
95
+ await _db_conn.execute("PRAGMA journal_mode=WAL")
96
+ await _db_conn.execute("PRAGMA foreign_keys=ON")
97
+ await _db_conn.execute("PRAGMA busy_timeout=5000")
98
+ await _migrate(_db_conn)
99
+ return _db_conn
100
+
101
+ async def _migrate(db: aiosqlite.Connection):
102
+ cur = await db.execute("PRAGMA user_version")
103
+ ver = (await cur.fetchone())[0]
104
  if ver < 1:
105
+ await db.executescript("""
106
+ CREATE TABLE IF NOT EXISTS files (
107
+ user_id INTEGER NOT NULL,
108
+ enc_path TEXT UNIQUE NOT NULL,
109
+ orig_name TEXT NOT NULL,
110
+ size INTEGER NOT NULL CHECK(size >= 0),
111
+ timestamp REAL NOT NULL DEFAULT (strftime('%s','now'))
112
+ );
113
+ CREATE INDEX IF NOT EXISTS idx_files_user ON files(user_id);
114
+ CREATE INDEX IF NOT EXISTS idx_files_name ON files(orig_name);
115
+
116
+ CREATE TABLE IF NOT EXISTS job_queue (
117
+ job_id TEXT PRIMARY KEY NOT NULL,
118
+ user_id INTEGER NOT NULL,
119
+ chat_id INTEGER NOT NULL,
120
+ file_id TEXT NOT NULL,
121
+ orig_filename TEXT NOT NULL,
122
+ expected_size INTEGER,
123
+ status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','processing','completed','failed','cancelled')),
124
+ created REAL NOT NULL DEFAULT (strftime('%s','now')),
125
+ updated REAL NOT NULL DEFAULT (strftime('%s','now'))
126
+ );
127
+ CREATE INDEX IF NOT EXISTS idx_jobs_status ON job_queue(status);
128
+
129
+ CREATE TABLE IF NOT EXISTS callback_tokens (
130
+ token TEXT PRIMARY KEY NOT NULL,
131
+ type TEXT NOT NULL,
132
+ data TEXT NOT NULL,
133
+ created REAL NOT NULL DEFAULT (strftime('%s','now'))
134
+ );
135
+
136
+ CREATE TABLE IF NOT EXISTS user_quota (
137
+ user_id INTEGER PRIMARY KEY NOT NULL,
138
+ used_bytes INTEGER NOT NULL DEFAULT 0 CHECK(used_bytes >= 0)
139
+ );
140
+ PRAGMA user_version = 1;
141
  """)
142
+ await db.commit()
 
 
 
143
 
144
  # ---------------------------------------------------------------------------
145
+ # DB helpers (all async via aiosqlite)
146
  # ---------------------------------------------------------------------------
 
 
147
  async def db_execute(sql: str, params: tuple = ()) -> None:
148
+ db = await get_db()
149
+ await db.execute(sql, params)
150
+ await db.commit()
151
+
152
+ async def db_fetchone(sql: str, params: tuple = ()) -> Optional[aiosqlite.Row]:
153
+ db = await get_db()
154
+ async with db.execute(sql, params) as cursor:
155
+ return await cursor.fetchone()
156
+
157
+ async def db_fetchall(sql: str, params: tuple = ()) -> list[aiosqlite.Row]:
158
+ db = await get_db()
159
+ async with db.execute(sql, params) as cursor:
160
+ return await cursor.fetchall()
 
 
 
 
 
 
 
 
 
 
 
161
 
162
  # ---------------------------------------------------------------------------
163
+ # Rate limiter (in‑memory)
164
  # ---------------------------------------------------------------------------
165
  class RateLimiter:
166
  def __init__(self):
 
173
  if user_id not in self._users:
174
  self._users[user_id] = []
175
  timestamps = self._users[user_id]
 
176
  self._users[user_id] = [t for t in timestamps if now - t < RATE_LIMIT_WINDOW]
177
  if len(self._users[user_id]) >= RATE_LIMIT_UPLOADS:
178
  return False
 
194
  return shutil.disk_usage("/").free
195
 
196
  def total_memory() -> int:
 
197
  try:
198
  with open("/sys/fs/cgroup/memory/memory.limit_in_bytes") as f:
199
  val = int(f.read().strip())
200
+ if val < 9_223_372_036_854_771_200:
201
  return val
202
  except: pass
 
203
  try:
204
  with open("/sys/fs/cgroup/memory.max") as f:
205
  val = f.read().strip()
206
  if val != "max":
207
  return int(val)
208
  except: pass
 
209
  try:
210
  with open("/proc/meminfo") as f:
211
  for line in f:
 
216
 
217
  def sanitize_filename(name: str) -> str:
218
  name = unicodedata.normalize('NFC', name)
 
219
  name = re.sub(r'[\x00-\x1f\x7f-\x9f]', '', name)
220
  name = re.sub(r'[\\/*?:"<>|]', '_', name)
 
221
  if len(name) > 200:
222
  stem, ext = os.path.splitext(name)
223
  stem = stem[:195 - len(ext)] + "..."
 
225
  return name or "unnamed"
226
 
227
  # ---------------------------------------------------------------------------
228
+ # FloodWait retry helper
229
  # ---------------------------------------------------------------------------
230
+ async def flood_retry(coro, *args, **kwargs):
231
+ """
232
+ Retry a Pyrogram method with exponential backoff on FloodWait.
233
+ `coro` should be a coroutine function (e.g., app.send_message).
234
+ """
235
  for attempt in range(5):
236
  try:
237
+ return await coro(*args, **kwargs)
 
 
 
238
  except pyro_errors.FloodWait as e:
239
  wait = e.value + 1
240
  logger.warning(f"FloodWait {wait}s")
241
  await asyncio.sleep(wait)
242
  except pyro_errors.RPCError as e:
243
+ if attempt == 4:
244
+ raise
245
  await asyncio.sleep(2 ** attempt)
246
  return None
247
 
248
  # ---------------------------------------------------------------------------
249
+ # Flask
250
  # ---------------------------------------------------------------------------
251
  web = Flask(__name__)
252
 
 
266
  web.run(host="0.0.0.0", port=7860, use_reloader=False)
267
 
268
  # ---------------------------------------------------------------------------
269
+ # Hugging Face async wrappers
270
  # ---------------------------------------------------------------------------
271
  hf = HfApi(token=HF_TOKEN)
272
 
 
298
  return await asyncio.to_thread(hf.list_repo_files, repo_id=DATASET_REPO, repo_type=DATASET_TYPE)
299
 
300
  # ---------------------------------------------------------------------------
301
+ # Quota (atomic with aiosqlite)
302
  # ---------------------------------------------------------------------------
 
 
 
 
 
 
303
  _quota_lock = asyncio.Lock()
304
 
305
  async def reserve_quota(user_id: int, bytes_: int) -> bool:
 
306
  async with _quota_lock:
307
  row = await db_fetchone("SELECT used_bytes FROM user_quota WHERE user_id=?", (user_id,))
308
  used = row["used_bytes"] if row else 0
 
321
  (bytes_, user_id)
322
  )
323
 
324
+ async def reconcile_quota():
 
325
  async with _quota_lock:
 
326
  await db_execute("DELETE FROM user_quota")
327
  rows = await db_fetchall("SELECT user_id, COALESCE(SUM(size),0) AS total FROM files GROUP BY user_id")
328
  for row in rows:
 
330
  (row["user_id"], row["total"]))
331
 
332
  # ---------------------------------------------------------------------------
333
+ # Job queue (atomic dequeue)
334
  # ---------------------------------------------------------------------------
335
  async def enqueue_job(user_id, chat_id, file_id, orig_filename, expected_size):
336
  job_id = secrets.token_hex(16)
 
341
  return job_id
342
 
343
  async def dequeue_job() -> Optional[Tuple[str, int, int, str, str, int]]:
344
+ """Atomically claim the next pending job."""
345
+ db = await get_db()
346
+ async with db.execute("BEGIN IMMEDIATE"):
347
+ cursor = await db.execute(
348
+ "SELECT job_id, user_id, chat_id, file_id, orig_filename, expected_size FROM job_queue WHERE status='pending' ORDER BY created LIMIT 1"
349
+ )
350
+ row = await cursor.fetchone()
351
+ if row:
352
+ await db.execute("UPDATE job_queue SET status='processing', updated=strftime('%s','now') WHERE job_id=?", (row["job_id"],))
353
+ await db.commit()
354
+ return row
 
 
 
 
 
355
 
356
  async def complete_job(job_id: str):
357
  await db_execute("DELETE FROM job_queue WHERE job_id=?", (job_id,))
358
 
 
 
 
359
  async def cancel_job(job_id: str):
360
+ await db_execute("UPDATE job_queue SET status='cancelled' WHERE job_id=? AND status IN ('pending','processing')", (job_id,))
361
 
362
  async def reset_processing_jobs():
363
  await db_execute("UPDATE job_queue SET status='pending' WHERE status='processing'")
364
 
365
  # ---------------------------------------------------------------------------
366
+ # Callback tokens
367
  # ---------------------------------------------------------------------------
368
  async def store_token(token: str, type_: str, data: dict):
369
  await db_execute("INSERT OR REPLACE INTO callback_tokens (token, type, data, created) VALUES (?,?,?,?)",
 
371
 
372
  async def get_token_data(token: str) -> Optional[dict]:
373
  row = await db_fetchone("SELECT data FROM callback_tokens WHERE token=? AND created > ?",
374
+ (token, time.time() - 3600))
375
  if row:
376
  return json.loads(row["data"])
377
  return None
 
380
  await db_execute("DELETE FROM callback_tokens WHERE token=?", (token,))
381
 
382
  # ---------------------------------------------------------------------------
383
+ # File listing helper (pagination)
384
  # ---------------------------------------------------------------------------
385
  async def build_file_list_message(user_id: int, page: int = 0, per_page: int = 10) -> Tuple[str, Optional[InlineKeyboardMarkup]]:
386
+ rows = await db_fetchall(
387
+ "SELECT enc_path, orig_name, size FROM files WHERE user_id=? ORDER BY timestamp DESC LIMIT ? OFFSET ?",
388
+ (user_id, per_page + 1, page * per_page)
389
+ )
390
  has_more = len(rows) > per_page
391
  if has_more:
392
  rows = rows[:per_page]
 
393
  if not rows and page == 0:
394
  return "You have no uploaded files.", None
395
 
 
402
  InlineKeyboardButton("🗑 Delete", callback_data=f"delete:{token}")
403
  ])
404
 
405
+ nav = []
406
  if page > 0:
407
+ nav.append(InlineKeyboardButton("⬅️ Prev", callback_data=f"files_page:{page-1}"))
408
  if has_more:
409
+ nav.append(InlineKeyboardButton("Next ➡️", callback_data=f"files_page:{page+1}"))
410
+ if nav:
411
+ buttons.append(nav)
412
 
413
  buttons.append([InlineKeyboardButton("⚠️ Delete All Files", callback_data="delete_all_confirm")])
414
  return "Your uploaded files:", InlineKeyboardMarkup(buttons)
415
 
416
  # ---------------------------------------------------------------------------
417
+ # Upload worker
418
  # ---------------------------------------------------------------------------
419
  active_uploads: dict[str, asyncio.Task] = {}
420
 
421
  async def process_upload(job_id: str, user_id: int, chat_id: int, file_id: str, orig_filename: str, expected_size: int):
 
422
  needed = (expected_size or 0) + MIN_FREE_DISK
423
+ while get_free_disk() < needed:
 
 
424
  logger.warning("Low disk, waiting...")
425
  await asyncio.sleep(30)
426
 
 
427
  if not await rate_limiter.is_allowed(user_id):
428
  await flood_retry(app.send_message, chat_id, "⚠️ Too many uploads, try later.")
429
  return
430
 
 
431
  try:
432
  info = await hf_dataset_info()
433
  current_used = info.used_storage or 0
434
  if current_used + (expected_size or 0) > DATASET_MAX:
435
+ await flood_retry(app.send_message, chat_id, "❌ Dataset is full.")
436
  return
437
  except Exception as e:
438
  logger.warning(f"Could not fetch dataset info: {e}")
 
452
  await msg.edit(f"❌ File too large ({format_size(file_size)}).")
453
  return
454
 
 
455
  if not await reserve_quota(user_id, file_size):
456
  await msg.edit(f"❌ Quota exceeded ({format_size(QUOTA_PER_USER)}).")
457
  return
 
460
  repo_path = f"{enc_folder(user_id)}/{enc_filename(safe_name)}"
461
  await msg.edit(f"⬆️ Uploading `{safe_name}` to Hugging Face...")
462
 
 
463
  for attempt in range(3):
464
  try:
465
  await hf_upload_file(local_path, repo_path)
 
467
  except hf_utils.HfHubHTTPError as e:
468
  if attempt == 2:
469
  raise
470
+ await asyncio.sleep(2 ** attempt)
 
 
471
 
 
472
  meta = await hf_get_metadata(repo_path)
473
  if meta.size != file_size:
 
474
  await hf_delete_file(repo_path)
475
  raise RuntimeError("Upload verification failed")
476
 
 
480
  logger.info(f"Uploaded {orig_filename} for user {user_id}")
481
 
482
  except asyncio.CancelledError:
 
483
  logger.info(f"Job {job_id} cancelled")
484
  if local_path and os.path.exists(local_path):
485
  os.remove(local_path)
 
505
  continue
506
  job_id, user_id, chat_id, file_id, orig_filename, expected_size = job
507
  logger.info(f"Worker {worker_id} processing {orig_filename}")
508
+ task = asyncio.create_task(process_upload(job_id, user_id, chat_id, file_id, orig_filename, expected_size))
 
 
509
  active_uploads[job_id] = task
510
  try:
511
  await task
 
516
  await complete_job(job_id)
517
 
518
  # ---------------------------------------------------------------------------
519
+ # Download helper
520
  # ---------------------------------------------------------------------------
521
  async def do_download(chat_id: int, enc_path: str, orig_name: str):
522
  temp_dir = tempfile.mkdtemp()
 
558
 
559
  @app.on_message(filters.command("start"))
560
  async def start_cmd(_, msg: Message):
561
+ await msg.reply("👋 Welcome! Send a file or t.me link.\n/files – list\n/status – stats")
562
 
563
  @app.on_message(filters.command("help"))
564
  async def help_cmd(_, msg: Message):
 
587
  @app.on_message(filters.command("files"))
588
  async def files_cmd(_, msg: Message):
589
  user_id = msg.from_user.id
 
590
  if not await db_fetchone("SELECT 1 FROM files WHERE user_id=?", (user_id,)):
591
  await reconcile_user(user_id)
592
+ text, markup = await build_file_list_message(user_id)
593
  if markup is None:
594
  await msg.reply(text)
595
  else:
 
600
  try:
601
  hf_files = await hf_list_files()
602
  hf_paths = [f for f in hf_files if f.startswith(enc_f + "/")]
603
+ db_rows = await db_fetchall("SELECT enc_path FROM files WHERE user_id=?", (user_id,))
604
+ db_paths = [r["enc_path"] for r in db_rows]
605
  for p in db_paths:
606
  if p not in hf_paths:
607
  await db_execute("DELETE FROM files WHERE enc_path=?", (p,))
 
623
 
624
  @app.on_message(filters.command("delete_all"))
625
  async def delete_all_cmd(_, msg: Message):
 
626
  kb = InlineKeyboardMarkup([[
627
  InlineKeyboardButton("Yes, delete all", callback_data="delete_all_confirm")
628
  ]])
629
+ await msg.reply("⚠️ Delete all your files? This cannot be undone.", reply_markup=kb)
630
 
631
  @app.on_message(filters.command("upload"))
632
  async def upload_cmd(_, msg: Message):
 
634
  if len(args) < 2:
635
  return await msg.reply("Usage: /upload <link>")
636
  link = args[1].strip()
637
+ match = re.search(r'(https?://t\.me/((?:c/)?[\w\d_]+)/(\d+))', link)
638
  if not match:
639
  return await msg.reply("Invalid link.")
640
+ chat_part, msg_id = match.group(2), int(match.group(3))
 
 
 
 
641
  try:
642
+ if chat_part.startswith("c/"):
643
+ chat_id = int("-100" + chat_part[2:])
644
+ else:
645
+ chat_id = chat_part
646
+ target = await app.get_messages(chat_id, msg_id)
647
  except Exception as e:
648
+ return await msg.reply(f"❌ {e}")
649
+ if not target.media:
650
  return await msg.reply("No file in that message.")
651
+ fid, size, name = _extract_media(target)
652
+ if not name: return
 
653
  if size and size > MAX_SINGLE_FILE:
654
  return await msg.reply(f"❌ File too large ({format_size(size)})")
655
+ await handle_upload_request(msg.from_user.id, msg.chat.id, fid, name, size)
656
 
657
  def _extract_media(msg: Message):
658
  if msg.document:
659
  return msg.document.file_id, msg.document.file_size, msg.document.file_name
660
+ if msg.video:
661
  return msg.video.file_id, msg.video.file_size, msg.video.file_name or f"{msg.video.file_unique_id}.mp4"
662
+ if msg.audio:
663
  return msg.audio.file_id, msg.audio.file_size, msg.audio.file_name or f"{msg.audio.file_unique_id}.mp3"
664
+ if msg.photo:
665
  return msg.photo.file_id, msg.photo.file_size, f"{msg.photo.file_unique_id}.jpg"
666
+ if msg.animation:
667
  return msg.animation.file_id, msg.animation.file_size, msg.animation.file_name or f"{msg.animation.file_unique_id}.mp4"
668
+ if msg.sticker:
669
+ ext = ".tgs" if msg.sticker.is_animated else ".webm" if msg.sticker.is_video else ".webp"
 
 
670
  return msg.sticker.file_id, msg.sticker.file_size, f"{msg.sticker.file_unique_id}{ext}"
671
+ if msg.video_note:
672
  return msg.video_note.file_id, msg.video_note.file_size, f"{msg.video_note.file_unique_id}.mp4"
673
+ if msg.voice:
674
  return msg.voice.file_id, msg.voice.file_size, f"{msg.voice.file_unique_id}.ogg"
675
  return None, None, None
676
 
677
  async def handle_upload_request(user_id, chat_id, file_id, filename, size):
 
678
  existing = await db_fetchone("SELECT enc_path FROM files WHERE user_id=? AND orig_name=? AND size=?",
679
  (user_id, filename, size))
680
  if existing:
 
693
  job_id = await enqueue_job(user_id, chat_id, file_id, filename, size)
694
  await flood_retry(app.send_message, chat_id, f"📦 `{filename}` queued.")
695
 
 
696
  # Callback handlers
 
697
  @app.on_callback_query(filters.regex("^get:"))
698
  async def get_callback(_, query: CallbackQuery):
699
  token = query.data.split(":", 1)[1]
700
  data = await get_token_data(token)
701
  if not data or data.get("user_id") != query.from_user.id:
702
+ return await query.answer("Invalid or expired.")
703
  enc_path = data["enc_path"]
704
  orig_name = enc_path.split("/")[-1]
705
  try: orig_name = decrypt_text(orig_name)
 
712
  token = query.data.split(":", 1)[1]
713
  data = await get_token_data(token)
714
  if not data or data.get("user_id") != query.from_user.id:
715
+ return await query.answer("Invalid or expired.")
716
  enc_path = data["enc_path"]
717
  try:
718
  await hf_delete_file(enc_path)
719
  except: pass
 
720
  row = await db_fetchone("SELECT size FROM files WHERE enc_path=?", (enc_path,))
721
  if row:
722
  await release_quota(query.from_user.id, row["size"])
723
  await db_execute("DELETE FROM files WHERE enc_path=?", (enc_path,))
724
  await delete_token(token)
725
  await query.answer("Deleted.")
726
+ text, markup = await build_file_list_message(query.from_user.id)
 
727
  await query.message.edit_text(text, reply_markup=markup)
728
 
729
  @app.on_callback_query(filters.regex("^delete_all_confirm$"))
730
+ async def delete_all_confirm(_, query: CallbackQuery):
731
  user_id = query.from_user.id
732
  rows = await db_fetchall("SELECT enc_path, size FROM files WHERE user_id=?", (user_id,))
733
  for r in rows:
 
738
  await query.message.edit_text("All files deleted.")
739
 
740
  @app.on_callback_query(filters.regex("^files_page:"))
741
+ async def files_page(_, query: CallbackQuery):
742
+ page = int(query.data.split(":")[1])
743
  text, markup = await build_file_list_message(query.from_user.id, page=page)
744
  await query.message.edit_text(text, reply_markup=markup)
745
  await query.answer()
746
 
747
  @app.on_callback_query(filters.regex("^dup_skip:"))
748
+ async def dup_skip(_, query: CallbackQuery):
749
+ await delete_token(query.data.split(":")[1])
 
750
  await query.message.edit_text("Skipped.")
751
 
752
  @app.on_callback_query(filters.regex("^dup_keep:"))
753
+ async def dup_keep(_, query: CallbackQuery):
754
+ token = query.data.split(":")[1]
755
  data = await get_token_data(token)
756
+ if not data: return await query.answer("Expired.")
757
+ stem, ext = os.path.splitext(data["filename"])
 
 
 
 
 
 
 
758
  new_name = f"{stem}_{secrets.token_hex(3)}{ext}"
759
+ job_id = await enqueue_job(data["user_id"], data["chat_id"], data["file_id"], new_name, data["size"])
760
  await query.message.edit_text(f"📦 Duplicate queued as `{new_name}`.")
761
  await delete_token(token)
762
 
763
  @app.on_callback_query(filters.regex("^cancel_job:"))
764
  async def cancel_job_callback(_, query: CallbackQuery):
765
+ job_id = query.data.split(":")[1]
766
  if job_id in active_uploads:
767
  active_uploads[job_id].cancel()
768
  await query.answer("Cancelling...")
769
  else:
770
+ await cancel_job(job_id)
771
+ await query.answer("Cancelled.")
772
 
773
+ # Album handling
 
 
774
  _album_cache: dict[str, dict] = {}
775
  _album_lock = asyncio.Lock()
776
 
777
  @app.on_message(filters.media & filters.media_group)
778
  async def album_handler(_, msg: Message):
779
+ if not msg.media_group_id: return
 
780
  group_id = str(msg.media_group_id)
781
  async with _album_lock:
782
  if group_id not in _album_cache:
 
795
  msgs = _album_cache[group_id]["msgs"]
796
  else:
797
  return
 
798
  files_info = []
799
  for m in msgs:
800
  fid, size, name = _extract_media(m)
 
802
  files_info.append((fid, size, name, m.id))
803
  if not files_info:
804
  return
805
+ oversized = [(fid, size, name) for fid, size, name, _ in files_info if size and size > MAX_SINGLE_FILE]
 
806
  if oversized:
807
+ names = ", ".join(n for _, _, n in oversized)
808
+ await flood_retry(app.send_message, msgs[0].chat.id, f"❌ Skipping {names} (exceeds {format_size(MAX_SINGLE_FILE)})")
809
+ files_info = [f for f in files_info if not (f[1] and f[1] > MAX_SINGLE_FILE)]
810
  if not files_info:
811
  return
 
812
  token = secrets.token_hex(8)
813
  await store_token(token, "album", {"user_id": msgs[0].from_user.id, "files": files_info})
814
  rows = []
 
822
  await flood_retry(app.send_message, msgs[0].chat.id,
823
  f"Album of {len(files_info)} files:\n{file_list}\nChoose:",
824
  reply_markup=InlineKeyboardMarkup(rows))
 
825
  async with _album_lock:
826
  if group_id in _album_cache:
827
  del _album_cache[group_id]
828
 
 
829
  async with _album_lock:
830
  _album_cache[group_id]["timer"] = asyncio.create_task(finalize())
831
 
 
835
  data = await get_token_data(token)
836
  if not data or data.get("user_id") != query.from_user.id:
837
  return await query.answer("Expired.")
838
+ for fid, size, name, _ in data["files"]:
839
  await handle_upload_request(data["user_id"], query.message.chat.id, fid, name, size)
840
  await query.message.edit_text("All files queued.")
841
  await delete_token(token)
 
851
  await delete_token(token)
852
 
853
  @app.on_callback_query(filters.regex("^cancel_album$"))
854
+ async def cancel_album(_, query: CallbackQuery):
855
  await query.message.delete()
856
 
857
+ # Direct file handler (non‑album)
 
 
858
  @app.on_message(filters.document | filters.video | filters.audio | filters.photo |
859
  filters.animation | filters.sticker | filters.video_note | filters.voice)
860
+ async def direct_file(_, msg: Message):
861
  if msg.media_group_id:
862
+ return
863
  fid, size, name = _extract_media(msg)
864
  if not name: return
865
  if size and size > MAX_SINGLE_FILE:
866
  return await msg.reply(f"❌ File too large ({format_size(size)}).")
867
  await handle_upload_request(msg.from_user.id, msg.chat.id, fid, name, size)
868
 
869
+ # Message link handler
 
 
870
  @app.on_message(filters.text & ~filters.command(["start","help","status","files","delete_all","upload"]) &
871
  filters.regex(r'https?://t\.me/'))
872
+ async def link_handler(_, msg: Message):
873
  match = re.search(r'(https?://t\.me/((?:c/)?[\w\d_]+)/(\d+))', msg.text)
874
  if not match: return
875
  chat_part, msg_id = match.group(2), int(match.group(3))
 
890
  await handle_upload_request(msg.from_user.id, msg.chat.id, fid, name, size)
891
 
892
  # ---------------------------------------------------------------------------
893
+ # Keep‑alive & startup
894
  # ---------------------------------------------------------------------------
895
  def keepalive():
896
  while True:
 
899
  except: pass
900
  time.sleep(KEEPALIVE_INTERVAL)
901
 
 
 
 
902
  bot_start_time = time.time()
903
 
904
  async def main():
905
+ # Startup recovery
906
  await reset_processing_jobs()
907
+ await reconcile_quota()
 
908
  for i in range(MAX_WORKERS):
909
  asyncio.create_task(upload_worker(i))
 
910
  async def token_cleanup():
911
  while True:
912
  await asyncio.sleep(3600)
913
  await db_execute("DELETE FROM callback_tokens WHERE created < ?", (time.time() - 3600,))
914
  asyncio.create_task(token_cleanup())
 
915
  await app.start()
916
  logger.info("Bot started")
917
+ # Wait forever
918
  stop_event = asyncio.Event()
919
  try:
920
  await stop_event.wait()
921
  except asyncio.CancelledError:
922
  pass
923
  finally:
 
 
 
924
  for t in active_uploads.values():
925
  t.cancel()
 
926
  await asyncio.sleep(1)
927
  await app.stop()
928
+ db = await get_db()
929
+ await db.close()
930
  logger.info("Bot stopped")
931
 
932
+ def shutdown_handler(loop):
 
933
  for task in asyncio.all_tasks(loop):
934
  task.cancel()
935
 
936
  if __name__ == "__main__":
937
  loop = asyncio.new_event_loop()
938
  asyncio.set_event_loop(loop)
 
939
  threading.Thread(target=run_web, daemon=True).start()
940
  threading.Thread(target=keepalive, daemon=True).start()
 
941
  for sig in (signal.SIGINT, signal.SIGTERM):
942
  try:
943
+ loop.add_signal_handler(sig, lambda: shutdown_handler(loop))
944
  except NotImplementedError:
 
945
  pass
946
  try:
947
  loop.run_until_complete(main())