Update app.py
Browse files
app.py
CHANGED
|
@@ -23,8 +23,8 @@ SPACE_HOST: str = "https://cxops-push.hf.space" # Keep‑alive URL
|
|
| 23 |
|
| 24 |
DATASET_REPO: str = "Cxops/Bzxv"
|
| 25 |
DATASET_TYPE: str = "dataset"
|
| 26 |
-
MAX_FILE_SIZE_TG: int = 1900 * 1024 * 1024 # 1900 MB per Telegram part
|
| 27 |
-
MAX_SINGLE_FILE_SIZE: int = 45 * 1024
|
| 28 |
MIN_FREE_DISK: int = 45 * 1024 * 1024 * 1024 # 45 GB
|
| 29 |
DATASET_MAX_SIZE: int = int(8.7 * 1024**4) # 8.7 TB
|
| 30 |
KEEPALIVE_INTERVAL: int = 300 # 5 minutes
|
|
@@ -120,7 +120,8 @@ def db_file_exists(user_id: int, orig_name: str) -> Optional[str]:
|
|
| 120 |
file_callback_map: dict[str, str] = {} # token → enc_path
|
| 121 |
upload_prompt_map: dict[str, List[Tuple]] = {} # token → list of entries
|
| 122 |
album_groups: dict[str, dict] = {}
|
| 123 |
-
|
|
|
|
| 124 |
|
| 125 |
def make_token() -> str:
|
| 126 |
return secrets.token_hex(8)
|
|
@@ -214,28 +215,31 @@ def extract_file_size(msg: Message) -> Optional[int]:
|
|
| 214 |
return msg.voice.file_size
|
| 215 |
return None
|
| 216 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
async def send_progress_updater(
|
| 218 |
job_id: int,
|
| 219 |
status_msg: Message,
|
| 220 |
cancel_event: asyncio.Event,
|
| 221 |
total_bytes: int,
|
| 222 |
-
filename: str = "",
|
| 223 |
) -> Tuple[asyncio.Task, list]:
|
| 224 |
"""
|
| 225 |
Update a Telegram message every 3 seconds with upload progress.
|
| 226 |
-
The filename is included in the header.
|
| 227 |
Returns (task, bytes_holder) – the caller must update bytes_holder[0] regularly.
|
| 228 |
"""
|
| 229 |
bytes_holder = [0]
|
| 230 |
start_time = time.time()
|
| 231 |
-
file_label = f"`{filename}` " if filename else ""
|
| 232 |
|
| 233 |
async def updater():
|
| 234 |
while not cancel_event.is_set():
|
| 235 |
elapsed = max(time.time() - start_time, 0.01)
|
| 236 |
current = bytes_holder[0]
|
| 237 |
speed = current / elapsed
|
| 238 |
-
text =
|
| 239 |
if total_bytes > 0:
|
| 240 |
pct = (100 * current / total_bytes)
|
| 241 |
text += f"{pct:.1f}% of {format_size(total_bytes)}\n"
|
|
@@ -280,14 +284,14 @@ async def do_upload_job(
|
|
| 280 |
) -> None:
|
| 281 |
"""
|
| 282 |
Download file from Telegram (by message or file_id) then upload to Hugging Face.
|
| 283 |
-
A timeout of UPLOAD_TIMEOUT is enforced
|
| 284 |
"""
|
| 285 |
temp_dir = tempfile.mkdtemp()
|
| 286 |
local_path = None
|
| 287 |
try:
|
| 288 |
# ---- Download from Telegram (progress shown) ----
|
| 289 |
await status_msg.edit_text(
|
| 290 |
-
|
| 291 |
reply_markup=InlineKeyboardMarkup(
|
| 292 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 293 |
),
|
|
@@ -309,7 +313,7 @@ async def do_upload_job(
|
|
| 309 |
elif total_size == 0 and expected_size and expected_size > 0:
|
| 310 |
total_size = expected_size
|
| 311 |
|
| 312 |
-
text =
|
| 313 |
if total_size > 0:
|
| 314 |
pct = (100 * current / total_size)
|
| 315 |
text += f"{pct:.1f}% of {format_size(total_size)}\n"
|
|
@@ -344,21 +348,13 @@ async def do_upload_job(
|
|
| 344 |
|
| 345 |
file_size = os.path.getsize(local_path)
|
| 346 |
|
| 347 |
-
# ---- Size limit check ----
|
| 348 |
-
if file_size > MAX_SINGLE_FILE_SIZE:
|
| 349 |
-
await status_msg.edit_text(
|
| 350 |
-
f"❌ File too large ({format_size(file_size)}). Maximum allowed is {format_size(MAX_SINGLE_FILE_SIZE)}.",
|
| 351 |
-
reply_markup=None,
|
| 352 |
-
)
|
| 353 |
-
return
|
| 354 |
-
|
| 355 |
# ---- Upload to Hugging Face (no progress, with timeout) ----
|
| 356 |
enc_folder = get_encrypted_folder(user_id)
|
| 357 |
enc_filename = get_encrypted_filename(orig_filename)
|
| 358 |
repo_path = f"{enc_folder}/{enc_filename}"
|
| 359 |
|
| 360 |
await status_msg.edit_text(
|
| 361 |
-
|
| 362 |
reply_markup=InlineKeyboardMarkup(
|
| 363 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 364 |
),
|
|
@@ -417,7 +413,7 @@ async def do_download_job(
|
|
| 417 |
loop = asyncio.get_event_loop()
|
| 418 |
|
| 419 |
await status_msg.edit_text(
|
| 420 |
-
|
| 421 |
reply_markup=InlineKeyboardMarkup(
|
| 422 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 423 |
),
|
|
@@ -445,7 +441,7 @@ async def do_download_job(
|
|
| 445 |
|
| 446 |
if total_size <= MAX_FILE_SIZE_TG:
|
| 447 |
prog_task, bytes_holder = await send_progress_updater(
|
| 448 |
-
job_id, status_msg, cancel_event, total_size
|
| 449 |
)
|
| 450 |
|
| 451 |
async def send_progress(current, total):
|
|
@@ -469,7 +465,7 @@ async def do_download_job(
|
|
| 469 |
await status_msg.edit_text("❌ Cancelled during send.")
|
| 470 |
return
|
| 471 |
await status_msg.edit_text(
|
| 472 |
-
f"📤 Sending part {i}/{num_parts}
|
| 473 |
reply_markup=InlineKeyboardMarkup(
|
| 474 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 475 |
),
|
|
@@ -533,7 +529,6 @@ async def delete_all_user_files(chat_id, user_id, query: CallbackQuery):
|
|
| 533 |
|
| 534 |
|
| 535 |
async def refresh_file_list(chat_id: int, user_id: int, message: Message):
|
| 536 |
-
"""Re‑fetch user files and edit the message with updated buttons."""
|
| 537 |
user_files = db_get_user_files(user_id)
|
| 538 |
if not user_files:
|
| 539 |
return await message.edit_text("You have no uploaded files.", reply_markup=None)
|
|
@@ -542,9 +537,8 @@ async def refresh_file_list(chat_id: int, user_id: int, message: Message):
|
|
| 542 |
for enc_path, orig_name, _ in user_files:
|
| 543 |
token = make_token()
|
| 544 |
file_callback_map[token] = enc_path
|
| 545 |
-
# Exactly as original: truncate to 40 characters
|
| 546 |
buttons.append([
|
| 547 |
-
InlineKeyboardButton(f"📥 {orig_name[:
|
| 548 |
InlineKeyboardButton("🗑 Delete", callback_data=f"delete:{token}"),
|
| 549 |
])
|
| 550 |
buttons.append([InlineKeyboardButton("⚠️ Delete All Files", callback_data=f"delete_all:{user_id}")])
|
|
@@ -565,10 +559,29 @@ async def process_album_group(media_group_id: str):
|
|
| 565 |
return
|
| 566 |
chat_id = group_data['chat_id']
|
| 567 |
user_id = group_data['user_id']
|
| 568 |
-
messages = group_data['messages'] # (message_id, filename, file_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 569 |
|
| 570 |
-
|
| 571 |
-
|
|
|
|
| 572 |
token = make_token()
|
| 573 |
upload_prompt_map[token] = [(user_id, msg_id, filename, file_id)]
|
| 574 |
kb = InlineKeyboardMarkup([
|
|
@@ -577,26 +590,36 @@ async def process_album_group(media_group_id: str):
|
|
| 577 |
])
|
| 578 |
await app.send_message(chat_id, f"Do you want to upload `{filename}`?", reply_markup=kb)
|
| 579 |
else:
|
| 580 |
-
files_list = "\n".join(f"• {fname}" for _, fname, _ in
|
| 581 |
token = make_token()
|
| 582 |
-
upload_prompt_map[token] = [(user_id, msg_id, fname, fid) for msg_id, fname, fid in
|
| 583 |
|
| 584 |
button_rows = [
|
| 585 |
[InlineKeyboardButton("⬆️ Upload All", callback_data=f"upload_all:{token}")]
|
| 586 |
]
|
| 587 |
-
for msg_id, fname, fid in
|
| 588 |
individual_token = make_token()
|
| 589 |
upload_prompt_map[individual_token] = [(user_id, msg_id, fname, fid)]
|
| 590 |
button_rows.append([InlineKeyboardButton(f"📤 {fname[:40]}", callback_data=f"upload:{individual_token}")])
|
| 591 |
button_rows.append([InlineKeyboardButton("❌ Cancel", callback_data="cancel_upload:0")])
|
| 592 |
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
|
| 597 |
|
| 598 |
# =============================================================================
|
| 599 |
-
# UPLOAD QUEUE WORKER
|
| 600 |
# =============================================================================
|
| 601 |
|
| 602 |
async def upload_worker():
|
|
@@ -606,7 +629,6 @@ async def upload_worker():
|
|
| 606 |
cancel_event = asyncio.Event()
|
| 607 |
active_jobs[job_id] = {"status": "running", "cancel_event": cancel_event}
|
| 608 |
|
| 609 |
-
# Wait until enough disk space (at least 45 GB free)
|
| 610 |
while get_free_disk() < MIN_FREE_DISK:
|
| 611 |
logger.warning("Low disk space – pausing uploads")
|
| 612 |
await asyncio.sleep(10)
|
|
@@ -644,16 +666,16 @@ async def cmd_help(_, message: Message):
|
|
| 644 |
text = (
|
| 645 |
"**Available Commands:**\n\n"
|
| 646 |
"/start – Welcome message\n"
|
| 647 |
-
"/files – List your uploaded files (download/delete
|
| 648 |
"/status – System & dataset space\n"
|
| 649 |
"/delete_all – Delete all your files (confirmation)\n"
|
| 650 |
"/upload <link> – Upload a file from a Telegram message link instantly\n"
|
| 651 |
"/help – This help\n\n"
|
| 652 |
"**Features:**\n"
|
|
|
|
| 653 |
"• Duplicate detection – you'll be asked to skip or keep a duplicate.\n"
|
| 654 |
-
"• File splitting for large
|
| 655 |
-
"•
|
| 656 |
-
"• Real‑time progress with filename, speed and ETA."
|
| 657 |
)
|
| 658 |
await message.reply_text(text, disable_web_page_preview=True)
|
| 659 |
|
|
@@ -712,12 +734,12 @@ async def cmd_files(_, message: Message):
|
|
| 712 |
token = make_token()
|
| 713 |
file_callback_map[token] = enc_path
|
| 714 |
buttons.append([
|
| 715 |
-
InlineKeyboardButton(f"📥 {orig_name[:
|
| 716 |
InlineKeyboardButton("🗑 Delete", callback_data=f"delete:{token}"),
|
| 717 |
])
|
| 718 |
buttons.append([InlineKeyboardButton("⚠️ Delete All Files", callback_data=f"delete_all:{user_id}")])
|
| 719 |
await message.reply_text(
|
| 720 |
-
"Your uploaded files:",
|
| 721 |
reply_markup=InlineKeyboardMarkup(buttons),
|
| 722 |
disable_web_page_preview=True,
|
| 723 |
)
|
|
@@ -784,7 +806,13 @@ async def cmd_upload(_, message: Message):
|
|
| 784 |
file_id = msg.voice.file_id
|
| 785 |
else:
|
| 786 |
filename = f"file_{msg_id}"
|
| 787 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 788 |
existing = db_file_exists(message.from_user.id, filename)
|
| 789 |
if existing:
|
| 790 |
token = make_token()
|
|
@@ -832,13 +860,14 @@ async def cb_dup_keep(_, query: CallbackQuery):
|
|
| 832 |
# Append a suffix like "_1" to make it unique
|
| 833 |
base, ext = os.path.splitext(data['orig_filename'])
|
| 834 |
new_filename = f"{base}_1{ext}"
|
|
|
|
| 835 |
job_id = data['job_id']
|
| 836 |
active_jobs[job_id] = {"status": "queued"}
|
| 837 |
await upload_queue.put((
|
| 838 |
data['user_id'],
|
| 839 |
data['chat_id'],
|
| 840 |
data['file_obj'],
|
| 841 |
-
new_filename,
|
| 842 |
job_id,
|
| 843 |
data['file_id'],
|
| 844 |
data['expected_size'],
|
|
@@ -903,7 +932,8 @@ async def cb_upload(_, query: CallbackQuery):
|
|
| 903 |
return await query.answer("Prompt expired.", show_alert=True)
|
| 904 |
for entry in entries:
|
| 905 |
user_id = entry[0]
|
| 906 |
-
if len(entry) == 4:
|
|
|
|
| 907 |
msg_id, filename, file_id = entry[1], entry[2], entry[3]
|
| 908 |
try:
|
| 909 |
file_msg = await app.get_messages(query.message.chat.id, msg_id)
|
|
@@ -915,6 +945,11 @@ async def cb_upload(_, query: CallbackQuery):
|
|
| 915 |
continue
|
| 916 |
file_obj = file_msg
|
| 917 |
expected_size = extract_file_size(file_msg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 918 |
# Duplicate check for local files
|
| 919 |
existing = db_file_exists(user_id, filename)
|
| 920 |
if existing:
|
|
@@ -935,9 +970,15 @@ async def cb_upload(_, query: CallbackQuery):
|
|
| 935 |
await query.message.reply_text(f"⚠️ File `{filename}` already exists. What do you want to do?", reply_markup=kb)
|
| 936 |
continue
|
| 937 |
else:
|
| 938 |
-
file_id, filename
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
file_obj = file_id
|
| 940 |
-
|
| 941 |
job_id = int(time.time() * 1000 + hash(filename))
|
| 942 |
active_jobs[job_id] = {"status": "queued"}
|
| 943 |
await upload_queue.put((user_id, query.message.chat.id, file_obj, filename, job_id, file_id, expected_size))
|
|
@@ -951,7 +992,8 @@ async def cb_upload_all(_, query: CallbackQuery):
|
|
| 951 |
return await query.answer("Prompt expired.", show_alert=True)
|
| 952 |
for entry in entries:
|
| 953 |
user_id = entry[0]
|
| 954 |
-
if len(entry) == 4:
|
|
|
|
| 955 |
msg_id, filename, file_id = entry[1], entry[2], entry[3]
|
| 956 |
try:
|
| 957 |
file_msg = await app.get_messages(query.message.chat.id, msg_id)
|
|
@@ -961,7 +1003,12 @@ async def cb_upload_all(_, query: CallbackQuery):
|
|
| 961 |
continue
|
| 962 |
file_obj = file_msg
|
| 963 |
expected_size = extract_file_size(file_msg)
|
| 964 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 965 |
existing = db_file_exists(user_id, filename)
|
| 966 |
if existing:
|
| 967 |
token2 = make_token()
|
|
@@ -981,9 +1028,14 @@ async def cb_upload_all(_, query: CallbackQuery):
|
|
| 981 |
await query.message.reply_text(f"⚠️ File `{filename}` already exists. What do you want to do?", reply_markup=kb)
|
| 982 |
continue
|
| 983 |
else:
|
| 984 |
-
file_id, filename
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 985 |
file_obj = file_id
|
| 986 |
-
|
| 987 |
job_id = int(time.time() * 1000 + hash(filename))
|
| 988 |
active_jobs[job_id] = {"status": "queued"}
|
| 989 |
await upload_queue.put((user_id, query.message.chat.id, file_obj, filename, job_id, file_id, expected_size))
|
|
@@ -1054,6 +1106,11 @@ async def on_message_link(client, message: Message):
|
|
| 1054 |
else:
|
| 1055 |
filename = f"file_{msg_id}"
|
| 1056 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1057 |
# Duplicate check for remote file
|
| 1058 |
existing = db_file_exists(message.from_user.id, filename)
|
| 1059 |
if existing:
|
|
@@ -1075,7 +1132,7 @@ async def on_message_link(client, message: Message):
|
|
| 1075 |
return
|
| 1076 |
|
| 1077 |
token = make_token()
|
| 1078 |
-
upload_prompt_map[token] = [(message.from_user.id, file_id, filename)]
|
| 1079 |
kb = InlineKeyboardMarkup([
|
| 1080 |
[InlineKeyboardButton("⬆️ Upload File", callback_data=f"upload:{token}")],
|
| 1081 |
[InlineKeyboardButton("❌ Cancel", callback_data="cancel_upload:0")],
|
|
@@ -1125,7 +1182,7 @@ async def on_direct_file(_, message: Message):
|
|
| 1125 |
else:
|
| 1126 |
return
|
| 1127 |
|
| 1128 |
-
# Album grouping
|
| 1129 |
if message.media_group_id and (message.photo or message.video or message.document):
|
| 1130 |
group_id = str(message.media_group_id)
|
| 1131 |
if group_id not in album_groups:
|
|
@@ -1135,7 +1192,7 @@ async def on_direct_file(_, message: Message):
|
|
| 1135 |
'messages': [],
|
| 1136 |
'timer': None,
|
| 1137 |
}
|
| 1138 |
-
album_groups[group_id]['messages'].append((message.id, filename, file_id))
|
| 1139 |
if album_groups[group_id]['timer']:
|
| 1140 |
album_groups[group_id]['timer'].cancel()
|
| 1141 |
|
|
@@ -1146,7 +1203,12 @@ async def on_direct_file(_, message: Message):
|
|
| 1146 |
album_groups[group_id]['timer'] = asyncio.create_task(timer_callback())
|
| 1147 |
return
|
| 1148 |
|
| 1149 |
-
# Single file –
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1150 |
existing = db_file_exists(message.from_user.id, filename)
|
| 1151 |
if existing:
|
| 1152 |
token = make_token()
|
|
|
|
| 23 |
|
| 24 |
DATASET_REPO: str = "Cxops/Bzxv"
|
| 25 |
DATASET_TYPE: str = "dataset"
|
| 26 |
+
MAX_FILE_SIZE_TG: int = 1900 * 1024 * 1024 # 1900 MB per Telegram part (splitting)
|
| 27 |
+
MAX_SINGLE_FILE_SIZE: int = 45 * 1024**3 # 45 GB – refuse larger uploads
|
| 28 |
MIN_FREE_DISK: int = 45 * 1024 * 1024 * 1024 # 45 GB
|
| 29 |
DATASET_MAX_SIZE: int = int(8.7 * 1024**4) # 8.7 TB
|
| 30 |
KEEPALIVE_INTERVAL: int = 300 # 5 minutes
|
|
|
|
| 120 |
file_callback_map: dict[str, str] = {} # token → enc_path
|
| 121 |
upload_prompt_map: dict[str, List[Tuple]] = {} # token → list of entries
|
| 122 |
album_groups: dict[str, dict] = {}
|
| 123 |
+
# Temporary storage for duplicate resolution
|
| 124 |
+
duplicate_prompt_map: dict[str, dict] = {} # token → {user_id, chat_id, file_msg_or_id, orig_filename, job_id, file_id, expected_size}
|
| 125 |
|
| 126 |
def make_token() -> str:
|
| 127 |
return secrets.token_hex(8)
|
|
|
|
| 215 |
return msg.voice.file_size
|
| 216 |
return None
|
| 217 |
|
| 218 |
+
def check_file_size_limit(filename: str, size: Optional[int]) -> Optional[str]:
|
| 219 |
+
"""Return error message if file exceeds 45 GB, else None."""
|
| 220 |
+
if size is not None and size > MAX_SINGLE_FILE_SIZE:
|
| 221 |
+
return f"❌ File `{filename}` is too large ({format_size(size)}). Maximum allowed is {format_size(MAX_SINGLE_FILE_SIZE)}."
|
| 222 |
+
return None
|
| 223 |
+
|
| 224 |
async def send_progress_updater(
|
| 225 |
job_id: int,
|
| 226 |
status_msg: Message,
|
| 227 |
cancel_event: asyncio.Event,
|
| 228 |
total_bytes: int,
|
|
|
|
| 229 |
) -> Tuple[asyncio.Task, list]:
|
| 230 |
"""
|
| 231 |
Update a Telegram message every 3 seconds with upload progress.
|
|
|
|
| 232 |
Returns (task, bytes_holder) – the caller must update bytes_holder[0] regularly.
|
| 233 |
"""
|
| 234 |
bytes_holder = [0]
|
| 235 |
start_time = time.time()
|
|
|
|
| 236 |
|
| 237 |
async def updater():
|
| 238 |
while not cancel_event.is_set():
|
| 239 |
elapsed = max(time.time() - start_time, 0.01)
|
| 240 |
current = bytes_holder[0]
|
| 241 |
speed = current / elapsed
|
| 242 |
+
text = "📤 Sending to Telegram...\n"
|
| 243 |
if total_bytes > 0:
|
| 244 |
pct = (100 * current / total_bytes)
|
| 245 |
text += f"{pct:.1f}% of {format_size(total_bytes)}\n"
|
|
|
|
| 284 |
) -> None:
|
| 285 |
"""
|
| 286 |
Download file from Telegram (by message or file_id) then upload to Hugging Face.
|
| 287 |
+
A timeout of UPLOAD_TIMEOUT is enforced.
|
| 288 |
"""
|
| 289 |
temp_dir = tempfile.mkdtemp()
|
| 290 |
local_path = None
|
| 291 |
try:
|
| 292 |
# ---- Download from Telegram (progress shown) ----
|
| 293 |
await status_msg.edit_text(
|
| 294 |
+
"⬇️ Downloading from Telegram...",
|
| 295 |
reply_markup=InlineKeyboardMarkup(
|
| 296 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 297 |
),
|
|
|
|
| 313 |
elif total_size == 0 and expected_size and expected_size > 0:
|
| 314 |
total_size = expected_size
|
| 315 |
|
| 316 |
+
text = "⬇️ Downloading from Telegram...\n"
|
| 317 |
if total_size > 0:
|
| 318 |
pct = (100 * current / total_size)
|
| 319 |
text += f"{pct:.1f}% of {format_size(total_size)}\n"
|
|
|
|
| 348 |
|
| 349 |
file_size = os.path.getsize(local_path)
|
| 350 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
# ---- Upload to Hugging Face (no progress, with timeout) ----
|
| 352 |
enc_folder = get_encrypted_folder(user_id)
|
| 353 |
enc_filename = get_encrypted_filename(orig_filename)
|
| 354 |
repo_path = f"{enc_folder}/{enc_filename}"
|
| 355 |
|
| 356 |
await status_msg.edit_text(
|
| 357 |
+
"⬆️ Uploading to Hugging Face...",
|
| 358 |
reply_markup=InlineKeyboardMarkup(
|
| 359 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 360 |
),
|
|
|
|
| 413 |
loop = asyncio.get_event_loop()
|
| 414 |
|
| 415 |
await status_msg.edit_text(
|
| 416 |
+
"⬇️ Downloading from dataset...",
|
| 417 |
reply_markup=InlineKeyboardMarkup(
|
| 418 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 419 |
),
|
|
|
|
| 441 |
|
| 442 |
if total_size <= MAX_FILE_SIZE_TG:
|
| 443 |
prog_task, bytes_holder = await send_progress_updater(
|
| 444 |
+
job_id, status_msg, cancel_event, total_size
|
| 445 |
)
|
| 446 |
|
| 447 |
async def send_progress(current, total):
|
|
|
|
| 465 |
await status_msg.edit_text("❌ Cancelled during send.")
|
| 466 |
return
|
| 467 |
await status_msg.edit_text(
|
| 468 |
+
f"📤 Sending part {i}/{num_parts}...",
|
| 469 |
reply_markup=InlineKeyboardMarkup(
|
| 470 |
[[InlineKeyboardButton("❌ Cancel", callback_data=f"cancel_job:{job_id}")]]
|
| 471 |
),
|
|
|
|
| 529 |
|
| 530 |
|
| 531 |
async def refresh_file_list(chat_id: int, user_id: int, message: Message):
|
|
|
|
| 532 |
user_files = db_get_user_files(user_id)
|
| 533 |
if not user_files:
|
| 534 |
return await message.edit_text("You have no uploaded files.", reply_markup=None)
|
|
|
|
| 537 |
for enc_path, orig_name, _ in user_files:
|
| 538 |
token = make_token()
|
| 539 |
file_callback_map[token] = enc_path
|
|
|
|
| 540 |
buttons.append([
|
| 541 |
+
InlineKeyboardButton(f"📥 {orig_name[:30]}", callback_data=f"get:{token}"),
|
| 542 |
InlineKeyboardButton("🗑 Delete", callback_data=f"delete:{token}"),
|
| 543 |
])
|
| 544 |
buttons.append([InlineKeyboardButton("⚠️ Delete All Files", callback_data=f"delete_all:{user_id}")])
|
|
|
|
| 559 |
return
|
| 560 |
chat_id = group_data['chat_id']
|
| 561 |
user_id = group_data['user_id']
|
| 562 |
+
messages = group_data['messages'] # list of (message_id, filename, file_id, size)
|
| 563 |
+
|
| 564 |
+
# Separate valid and too-large files
|
| 565 |
+
valid = []
|
| 566 |
+
skipped = []
|
| 567 |
+
for msg_id, filename, file_id, size in messages:
|
| 568 |
+
limit_msg = check_file_size_limit(filename, size)
|
| 569 |
+
if limit_msg:
|
| 570 |
+
skipped.append(filename)
|
| 571 |
+
else:
|
| 572 |
+
valid.append((msg_id, filename, file_id, size))
|
| 573 |
+
|
| 574 |
+
if not valid:
|
| 575 |
+
skip_text = "\n".join(f"• `{f}`" for f in skipped)
|
| 576 |
+
await app.send_message(
|
| 577 |
+
chat_id,
|
| 578 |
+
f"❌ All files exceed the {format_size(MAX_SINGLE_FILE_SIZE)} limit and were skipped:\n{skip_text}"
|
| 579 |
+
)
|
| 580 |
+
return
|
| 581 |
|
| 582 |
+
# Build prompt for valid files
|
| 583 |
+
if len(valid) == 1:
|
| 584 |
+
msg_id, filename, file_id, _ = valid[0]
|
| 585 |
token = make_token()
|
| 586 |
upload_prompt_map[token] = [(user_id, msg_id, filename, file_id)]
|
| 587 |
kb = InlineKeyboardMarkup([
|
|
|
|
| 590 |
])
|
| 591 |
await app.send_message(chat_id, f"Do you want to upload `{filename}`?", reply_markup=kb)
|
| 592 |
else:
|
| 593 |
+
files_list = "\n".join(f"• {fname}" for _, fname, _, _ in valid)
|
| 594 |
token = make_token()
|
| 595 |
+
upload_prompt_map[token] = [(user_id, msg_id, fname, fid) for msg_id, fname, fid, _ in valid]
|
| 596 |
|
| 597 |
button_rows = [
|
| 598 |
[InlineKeyboardButton("⬆️ Upload All", callback_data=f"upload_all:{token}")]
|
| 599 |
]
|
| 600 |
+
for msg_id, fname, fid, _ in valid:
|
| 601 |
individual_token = make_token()
|
| 602 |
upload_prompt_map[individual_token] = [(user_id, msg_id, fname, fid)]
|
| 603 |
button_rows.append([InlineKeyboardButton(f"📤 {fname[:40]}", callback_data=f"upload:{individual_token}")])
|
| 604 |
button_rows.append([InlineKeyboardButton("❌ Cancel", callback_data="cancel_upload:0")])
|
| 605 |
|
| 606 |
+
prompt_text = f"Received {len(valid)} files.\n{files_list}\n\nChoose:"
|
| 607 |
+
if skipped:
|
| 608 |
+
prompt_text += f"\n⚠️ Skipped (exceed {format_size(MAX_SINGLE_FILE_SIZE)}): " + ", ".join(f"`{f}`" for f in skipped)
|
| 609 |
+
|
| 610 |
+
await app.send_message(chat_id, prompt_text, reply_markup=InlineKeyboardMarkup(button_rows))
|
| 611 |
+
|
| 612 |
+
# Inform about skipped files separately if necessary
|
| 613 |
+
if skipped and len(valid) == 1:
|
| 614 |
+
# Already sent the single upload prompt, we can send a quick note about skipped
|
| 615 |
+
await app.send_message(
|
| 616 |
+
chat_id,
|
| 617 |
+
f"⚠️ Skipped {len(skipped)} file(s) because they exceed {format_size(MAX_SINGLE_FILE_SIZE)}: " + ", ".join(f"`{f}`" for f in skipped)
|
| 618 |
+
)
|
| 619 |
|
| 620 |
|
| 621 |
# =============================================================================
|
| 622 |
+
# UPLOAD QUEUE WORKER
|
| 623 |
# =============================================================================
|
| 624 |
|
| 625 |
async def upload_worker():
|
|
|
|
| 629 |
cancel_event = asyncio.Event()
|
| 630 |
active_jobs[job_id] = {"status": "running", "cancel_event": cancel_event}
|
| 631 |
|
|
|
|
| 632 |
while get_free_disk() < MIN_FREE_DISK:
|
| 633 |
logger.warning("Low disk space – pausing uploads")
|
| 634 |
await asyncio.sleep(10)
|
|
|
|
| 666 |
text = (
|
| 667 |
"**Available Commands:**\n\n"
|
| 668 |
"/start – Welcome message\n"
|
| 669 |
+
"/files – List your uploaded files (download/delete)\n"
|
| 670 |
"/status – System & dataset space\n"
|
| 671 |
"/delete_all – Delete all your files (confirmation)\n"
|
| 672 |
"/upload <link> – Upload a file from a Telegram message link instantly\n"
|
| 673 |
"/help – This help\n\n"
|
| 674 |
"**Features:**\n"
|
| 675 |
+
"• Maximum file size: 45 GB – larger uploads are rejected.\n"
|
| 676 |
"• Duplicate detection – you'll be asked to skip or keep a duplicate.\n"
|
| 677 |
+
"• File splitting for large downloads (>1.9 GB).\n"
|
| 678 |
+
"• Real‑time progress with speed and ETA."
|
|
|
|
| 679 |
)
|
| 680 |
await message.reply_text(text, disable_web_page_preview=True)
|
| 681 |
|
|
|
|
| 734 |
token = make_token()
|
| 735 |
file_callback_map[token] = enc_path
|
| 736 |
buttons.append([
|
| 737 |
+
InlineKeyboardButton(f"📥 {orig_name[:25]}", callback_data=f"get:{token}"),
|
| 738 |
InlineKeyboardButton("🗑 Delete", callback_data=f"delete:{token}"),
|
| 739 |
])
|
| 740 |
buttons.append([InlineKeyboardButton("⚠️ Delete All Files", callback_data=f"delete_all:{user_id}")])
|
| 741 |
await message.reply_text(
|
| 742 |
+
"Your uploaded files (click to download, delete to remove):",
|
| 743 |
reply_markup=InlineKeyboardMarkup(buttons),
|
| 744 |
disable_web_page_preview=True,
|
| 745 |
)
|
|
|
|
| 806 |
file_id = msg.voice.file_id
|
| 807 |
else:
|
| 808 |
filename = f"file_{msg_id}"
|
| 809 |
+
|
| 810 |
+
# Check file size limit first
|
| 811 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 812 |
+
if limit_err:
|
| 813 |
+
return await message.reply_text(limit_err)
|
| 814 |
+
|
| 815 |
+
# Check for duplicate
|
| 816 |
existing = db_file_exists(message.from_user.id, filename)
|
| 817 |
if existing:
|
| 818 |
token = make_token()
|
|
|
|
| 860 |
# Append a suffix like "_1" to make it unique
|
| 861 |
base, ext = os.path.splitext(data['orig_filename'])
|
| 862 |
new_filename = f"{base}_1{ext}"
|
| 863 |
+
# Queue with the modified filename
|
| 864 |
job_id = data['job_id']
|
| 865 |
active_jobs[job_id] = {"status": "queued"}
|
| 866 |
await upload_queue.put((
|
| 867 |
data['user_id'],
|
| 868 |
data['chat_id'],
|
| 869 |
data['file_obj'],
|
| 870 |
+
new_filename, # use modified name
|
| 871 |
job_id,
|
| 872 |
data['file_id'],
|
| 873 |
data['expected_size'],
|
|
|
|
| 932 |
return await query.answer("Prompt expired.", show_alert=True)
|
| 933 |
for entry in entries:
|
| 934 |
user_id = entry[0]
|
| 935 |
+
if len(entry) == 4 and isinstance(entry[1], int) and entry[1] > 0:
|
| 936 |
+
# Local file: (user_id, msg_id, filename, file_id)
|
| 937 |
msg_id, filename, file_id = entry[1], entry[2], entry[3]
|
| 938 |
try:
|
| 939 |
file_msg = await app.get_messages(query.message.chat.id, msg_id)
|
|
|
|
| 945 |
continue
|
| 946 |
file_obj = file_msg
|
| 947 |
expected_size = extract_file_size(file_msg)
|
| 948 |
+
# Check size limit
|
| 949 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 950 |
+
if limit_err:
|
| 951 |
+
await query.message.reply_text(limit_err)
|
| 952 |
+
continue
|
| 953 |
# Duplicate check for local files
|
| 954 |
existing = db_file_exists(user_id, filename)
|
| 955 |
if existing:
|
|
|
|
| 970 |
await query.message.reply_text(f"⚠️ File `{filename}` already exists. What do you want to do?", reply_markup=kb)
|
| 971 |
continue
|
| 972 |
else:
|
| 973 |
+
# Remote file: (user_id, file_id, filename, expected_size)
|
| 974 |
+
file_id, filename, expected_size = entry[1], entry[2], entry[3] if len(entry) == 4 else (entry[1], entry[2], None)
|
| 975 |
+
# Check size limit
|
| 976 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 977 |
+
if limit_err:
|
| 978 |
+
await query.message.reply_text(limit_err)
|
| 979 |
+
continue
|
| 980 |
file_obj = file_id
|
| 981 |
+
|
| 982 |
job_id = int(time.time() * 1000 + hash(filename))
|
| 983 |
active_jobs[job_id] = {"status": "queued"}
|
| 984 |
await upload_queue.put((user_id, query.message.chat.id, file_obj, filename, job_id, file_id, expected_size))
|
|
|
|
| 992 |
return await query.answer("Prompt expired.", show_alert=True)
|
| 993 |
for entry in entries:
|
| 994 |
user_id = entry[0]
|
| 995 |
+
if len(entry) == 4 and isinstance(entry[1], int) and entry[1] > 0:
|
| 996 |
+
# Local file
|
| 997 |
msg_id, filename, file_id = entry[1], entry[2], entry[3]
|
| 998 |
try:
|
| 999 |
file_msg = await app.get_messages(query.message.chat.id, msg_id)
|
|
|
|
| 1003 |
continue
|
| 1004 |
file_obj = file_msg
|
| 1005 |
expected_size = extract_file_size(file_msg)
|
| 1006 |
+
# Check size limit
|
| 1007 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 1008 |
+
if limit_err:
|
| 1009 |
+
await query.message.reply_text(limit_err)
|
| 1010 |
+
continue
|
| 1011 |
+
# Duplicate check
|
| 1012 |
existing = db_file_exists(user_id, filename)
|
| 1013 |
if existing:
|
| 1014 |
token2 = make_token()
|
|
|
|
| 1028 |
await query.message.reply_text(f"⚠️ File `{filename}` already exists. What do you want to do?", reply_markup=kb)
|
| 1029 |
continue
|
| 1030 |
else:
|
| 1031 |
+
# Remote file: (user_id, file_id, filename, expected_size)
|
| 1032 |
+
file_id, filename, expected_size = entry[1], entry[2], entry[3] if len(entry) == 4 else (entry[1], entry[2], None)
|
| 1033 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 1034 |
+
if limit_err:
|
| 1035 |
+
await query.message.reply_text(limit_err)
|
| 1036 |
+
continue
|
| 1037 |
file_obj = file_id
|
| 1038 |
+
|
| 1039 |
job_id = int(time.time() * 1000 + hash(filename))
|
| 1040 |
active_jobs[job_id] = {"status": "queued"}
|
| 1041 |
await upload_queue.put((user_id, query.message.chat.id, file_obj, filename, job_id, file_id, expected_size))
|
|
|
|
| 1106 |
else:
|
| 1107 |
filename = f"file_{msg_id}"
|
| 1108 |
|
| 1109 |
+
# Check file size limit first
|
| 1110 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 1111 |
+
if limit_err:
|
| 1112 |
+
return await message.reply_text(limit_err)
|
| 1113 |
+
|
| 1114 |
# Duplicate check for remote file
|
| 1115 |
existing = db_file_exists(message.from_user.id, filename)
|
| 1116 |
if existing:
|
|
|
|
| 1132 |
return
|
| 1133 |
|
| 1134 |
token = make_token()
|
| 1135 |
+
upload_prompt_map[token] = [(message.from_user.id, file_id, filename, expected_size)]
|
| 1136 |
kb = InlineKeyboardMarkup([
|
| 1137 |
[InlineKeyboardButton("⬆️ Upload File", callback_data=f"upload:{token}")],
|
| 1138 |
[InlineKeyboardButton("❌ Cancel", callback_data="cancel_upload:0")],
|
|
|
|
| 1182 |
else:
|
| 1183 |
return
|
| 1184 |
|
| 1185 |
+
# Album grouping – store size along with other info
|
| 1186 |
if message.media_group_id and (message.photo or message.video or message.document):
|
| 1187 |
group_id = str(message.media_group_id)
|
| 1188 |
if group_id not in album_groups:
|
|
|
|
| 1192 |
'messages': [],
|
| 1193 |
'timer': None,
|
| 1194 |
}
|
| 1195 |
+
album_groups[group_id]['messages'].append((message.id, filename, file_id, expected_size))
|
| 1196 |
if album_groups[group_id]['timer']:
|
| 1197 |
album_groups[group_id]['timer'].cancel()
|
| 1198 |
|
|
|
|
| 1203 |
album_groups[group_id]['timer'] = asyncio.create_task(timer_callback())
|
| 1204 |
return
|
| 1205 |
|
| 1206 |
+
# Single file – check size limit first
|
| 1207 |
+
limit_err = check_file_size_limit(filename, expected_size)
|
| 1208 |
+
if limit_err:
|
| 1209 |
+
return await message.reply_text(limit_err)
|
| 1210 |
+
|
| 1211 |
+
# Duplicate check
|
| 1212 |
existing = db_file_exists(message.from_user.id, filename)
|
| 1213 |
if existing:
|
| 1214 |
token = make_token()
|