diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -1,28 +1,26 @@ + import os import hmac import hashlib import json -import shutil -import threading -import time -import uuid -import logging -from datetime import datetime -from io import BytesIO from urllib.parse import unquote, parse_qsl, urlencode -from typing import Union, Optional - -import requests from flask import Flask, request, jsonify, Response, send_file from flask_caching import Cache +import logging +import threading +import time +from datetime import datetime from huggingface_hub import HfApi, hf_hub_download, utils as hf_utils from werkzeug.utils import secure_filename +import requests +from io import BytesIO +import uuid +from typing import Union, Optional, Dict, Any, List, Tuple app = Flask(__name__) app.secret_key = os.getenv("FLASK_SECRET_KEY", "supersecretkey_mini_app_unique_v2") BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '6750208873:AAE2hvPlJ99dBdhGa_Brre0IIpUdOvXxHt4') -DATA_FILE = 'cloudeng_mini_app_data.json' -DATA_FILE_BACKUP = DATA_FILE + '.bak' +DATA_FILE = 'cloudeng_mini_app_data_v2.json' REPO_ID = "Eluza133/Z1e1u" HF_TOKEN_WRITE = os.getenv("HF_TOKEN") HF_TOKEN_READ = os.getenv("HF_TOKEN_READ") or HF_TOKEN_WRITE @@ -33,9 +31,10 @@ cache = Cache(app, config={'CACHE_TYPE': 'simple'}) logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') AUTH_DATA_LIFETIME = 3600 -data_lock = threading.Lock() -def find_node_by_id(filesystem, node_id): +# --- Filesystem Utilities --- + +def find_node_by_id(filesystem: Dict[str, Any], node_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: if not filesystem or not isinstance(filesystem, dict): return None, None if filesystem.get('id') == node_id: @@ -46,41 +45,65 @@ def find_node_by_id(filesystem, node_id): while queue: current_node, parent = queue.pop(0) - if current_node.get('type') == 'folder' and 'children' in current_node: - for child in current_node.get('children', []): + node_type = current_node.get('type') + children = current_node.get('children') + + if node_type == 'folder' and isinstance(children, list): + for child in children: + if not isinstance(child, dict): continue child_id = child.get('id') if not child_id: continue if child_id == node_id: return child, current_node - if child_id not in visited and isinstance(child, dict) and child.get('type') == 'folder': + + if child.get('type') == 'folder' and child_id not in visited: visited.add(child_id) queue.append((child, current_node)) return None, None -def add_node(filesystem, parent_id, node_data): +def add_node(filesystem: Dict[str, Any], parent_id: str, node_data: Dict[str, Any]) -> bool: parent_node, _ = find_node_by_id(filesystem, parent_id) if parent_node and parent_node.get('type') == 'folder': if 'children' not in parent_node or not isinstance(parent_node['children'], list): parent_node['children'] = [] + existing_ids = {child.get('id') for child in parent_node['children'] if isinstance(child, dict)} - if node_data.get('id') not in existing_ids: + new_node_id = node_data.get('id') + if new_node_id and new_node_id not in existing_ids: parent_node['children'].append(node_data) return True + elif not new_node_id: + logging.error(f"Attempted to add node without ID to parent {parent_id}") + else: + logging.warning(f"Node with ID {new_node_id} already exists under parent {parent_id}") + + elif not parent_node: + logging.error(f"Parent node {parent_id} not found for adding node.") + elif parent_node.get('type') != 'folder': + logging.error(f"Cannot add node to non-folder parent {parent_id} (type: {parent_node.get('type')}).") + return False -def remove_node(filesystem, node_id): +def remove_node(filesystem: Dict[str, Any], node_id: str) -> bool: + if node_id == filesystem.get('id'): + logging.error("Attempted to remove root node.") + return False + node_to_remove, parent_node = find_node_by_id(filesystem, node_id) - if node_to_remove and parent_node and 'children' in parent_node and isinstance(parent_node['children'], list): + + if node_to_remove and parent_node and isinstance(parent_node.get('children'), list): original_length = len(parent_node['children']) parent_node['children'] = [child for child in parent_node['children'] if not isinstance(child, dict) or child.get('id') != node_id] return len(parent_node['children']) < original_length - if node_to_remove and node_id == filesystem.get('id'): - logging.warning("Attempted to remove root node directly.") - return False + elif node_to_remove and not parent_node: + logging.error(f"Found node {node_id} but it has no parent (should not happen except for root).") + elif not node_to_remove: + logging.warning(f"Node {node_id} not found for removal.") + return False -def get_node_path_list(filesystem, node_id): +def get_node_path_list(filesystem: Dict[str, Any], node_id: str) -> List[Dict[str, str]]: path_list = [] current_id = node_id processed_ids = set() @@ -91,14 +114,14 @@ def get_node_path_list(filesystem, node_id): processed_ids.add(current_id) depth += 1 node, parent = find_node_by_id(filesystem, current_id) - if not node or not isinstance(node, dict): - logging.warning(f"Path traversal stopped: Node not found or invalid for ID {current_id}") + if not node: + logging.warning(f"Path traversal broken: Node {current_id} not found.") break - path_list.append({ - 'id': node.get('id'), - 'name': node.get('name', node.get('original_filename', 'Unknown')) - }) - if not parent or not isinstance(parent, dict): + + node_name = node.get('name', node.get('original_filename', 'Unknown')) + path_list.append({'id': node.get('id'), 'name': node_name}) + + if not parent: break parent_id = parent.get('id') if parent_id == current_id: @@ -106,142 +129,150 @@ def get_node_path_list(filesystem, node_id): break current_id = parent_id - if not any(p['id'] == 'root' for p in path_list) and filesystem and filesystem.get('id') == 'root': - path_list.append({'id': 'root', 'name': filesystem.get('name','Root')}) + if depth >= max_depth: + logging.error(f"Max path depth reached for node {node_id}. Returning partial path.") + + # Ensure root is always present if not found during traversal + if not any(p['id'] == 'root' for p in path_list): + # Check if filesystem itself is the root + root_node, _ = find_node_by_id(filesystem, 'root') + if root_node: + path_list.append({'id': 'root', 'name': root_node.get('name', 'Root')}) + else: + # Fallback if even the root node lookup fails + path_list.append({'id': 'root', 'name': 'Root'}) + final_path = [] seen_ids = set() for item in reversed(path_list): - if item['id'] not in seen_ids: + item_id = item.get('id') + if item_id and item_id not in seen_ids: final_path.append(item) - seen_ids.add(item['id']) - if not final_path or final_path[0]['id'] != 'root': - final_path.insert(0, {'id': 'root', 'name': filesystem.get('name','Root') if filesystem else 'Root'}) + seen_ids.add(item_id) - return final_path + # If the final path is empty or doesn't start with root, force root at the beginning + if not final_path or final_path[0].get('id') != 'root': + root_node, _ = find_node_by_id(filesystem, 'root') + root_name = root_node.get('name', 'Root') if root_node else 'Root' + final_path.insert(0, {'id': 'root', 'name': root_name}) + # Remove potential duplicate roots further down + final_path = [p for i, p in enumerate(final_path) if p.get('id') != 'root' or i == 0] -def initialize_user_filesystem(user_data): - if 'filesystem' not in user_data or not isinstance(user_data.get('filesystem'), dict) or not user_data['filesystem'].get('id') == 'root': - logging.warning(f"Initializing/Resetting filesystem for user.") + return final_path + +def initialize_user_filesystem(user_data: Dict[str, Any]): + if 'filesystem' not in user_data or not isinstance(user_data['filesystem'], dict) or user_data['filesystem'].get('id') != 'root': user_data['filesystem'] = { "type": "folder", "id": "root", "name": "Root", "children": [] } - elif 'children' not in user_data['filesystem'] or not isinstance(user_data['filesystem']['children'], list): - user_data['filesystem']['children'] = [] +# --- Data Loading/Saving --- -def load_data_from_file(filepath): +@cache.memoize(timeout=60) +def load_data() -> Dict[str, Any]: try: - with open(filepath, 'r', encoding='utf-8') as file: + download_db_from_hf() + if not os.path.exists(DATA_FILE) or os.path.getsize(DATA_FILE) == 0: + logging.warning(f"{DATA_FILE} is missing or empty after download attempt. Initializing.") + return {'users': {}} + + with open(DATA_FILE, 'r', encoding='utf-8') as file: data = json.load(file) if not isinstance(data, dict): - logging.warning(f"Data file {filepath} is not a dict, treating as invalid.") - return None + logging.error(f"Data file {DATA_FILE} is not a dict. Initializing empty.") + return {'users': {}} + data.setdefault('users', {}) - for user_id, user_data in data['users'].items(): - if isinstance(user_data, dict): - initialize_user_filesystem(user_data) - else: - logging.warning(f"Invalid user_data structure for user {user_id} in {filepath}, skipping.") - logging.info(f"Data loaded successfully from {filepath}.") + if not isinstance(data['users'], dict): + logging.error(f"'users' key in {DATA_FILE} is not a dict. Resetting.") + data['users'] = {} + + for user_id, user_data in list(data['users'].items()): # Use list to allow modification + if not isinstance(user_data, dict): + logging.warning(f"Invalid data type for user {user_id}. Removing entry.") + del data['users'][user_id] + continue + initialize_user_filesystem(user_data) + + logging.info(f"Data loaded successfully from {DATA_FILE}.") return data except FileNotFoundError: - logging.info(f"{filepath} not found locally.") - return None - except json.JSONDecodeError: - logging.error(f"Error decoding JSON from {filepath}.") - return None + logging.warning(f"{DATA_FILE} not found locally. Initializing empty data.") + return {'users': {}} + except json.JSONDecodeError as e: + logging.error(f"Error decoding JSON from {DATA_FILE}: {e}. Returning empty data.") + # Potentially try to recover from backup or previous version here if needed + return {'users': {}} except Exception as e: - logging.error(f"Error loading data from {filepath}: {e}") - return None - -@cache.memoize(timeout=60) -def load_data(): - with data_lock: - data = None - primary_exists = os.path.exists(DATA_FILE) - backup_exists = os.path.exists(DATA_FILE_BACKUP) - - if primary_exists: - data = load_data_from_file(DATA_FILE) - - if data is None and backup_exists: - logging.warning(f"Primary data file {DATA_FILE} failed to load or missing, attempting backup.") - data = load_data_from_file(DATA_FILE_BACKUP) - if data: - logging.info("Loaded data from backup. Attempting to restore primary file.") - try: - shutil.copy2(DATA_FILE_BACKUP, DATA_FILE) - except Exception as e: - logging.error(f"Failed to restore primary file from backup: {e}") - - if data is None: - logging.warning("Both primary and backup data files failed to load or missing. Attempting download from HF.") - download_success = download_db_from_hf() - if download_success: - data = load_data_from_file(DATA_FILE) - - if data is None: - logging.critical("CRITICAL: Could not load data from local files or HF. Initializing empty data structure.") - data = {'users': {}} + logging.error(f"Unexpected error loading data: {e}", exc_info=True) + return {'users': {}} - return data - - -def save_data(data): - with data_lock: - try: - if os.path.exists(DATA_FILE): - try: - shutil.copy2(DATA_FILE, DATA_FILE_BACKUP) - logging.info(f"Created backup: {DATA_FILE_BACKUP}") - except Exception as backup_err: - logging.error(f"Failed to create backup file {DATA_FILE_BACKUP}: {backup_err}") - - with open(DATA_FILE, 'w', encoding='utf-8') as file: - json.dump(data, file, ensure_ascii=False, indent=2) # Use indent=2 for smaller file size - - logging.info(f"Data saved locally to {DATA_FILE}") - cache.clear() - upload_db_to_hf() - return True - - except Exception as e: - logging.error(f"CRITICAL: Error saving data to {DATA_FILE}: {e}") - return False +def save_data(data: Dict[str, Any]): + temp_file = DATA_FILE + ".tmp" + try: + with open(temp_file, 'w', encoding='utf-8') as file: + json.dump(data, file, ensure_ascii=False, indent=4) + os.replace(temp_file, DATA_FILE) + cache.clear() + logging.info(f"Data saved successfully to {DATA_FILE}.") + # Trigger async upload AFTER successful local save + upload_db_to_hf_async() + except Exception as e: + logging.error(f"Error saving data to {DATA_FILE}: {e}", exc_info=True) + if os.path.exists(temp_file): + try: + os.remove(temp_file) + except OSError as rm_err: + logging.error(f"Failed to remove temporary save file {temp_file}: {rm_err}") + # Re-raise the exception to signal failure to the caller + raise RuntimeError(f"Failed to save data: {e}") from e -def upload_db_to_hf(): +def upload_db_to_hf_async(): if not HF_TOKEN_WRITE: logging.warning("HF_TOKEN_WRITE not set, skipping database upload.") return - if not os.path.exists(DATA_FILE): - logging.warning(f"Local data file {DATA_FILE} not found for upload.") - return - try: - api = HfApi() - api.upload_file( - path_or_fileobj=DATA_FILE, - path_in_repo=DATA_FILE, - repo_id=REPO_ID, - repo_type="dataset", - token=HF_TOKEN_WRITE, - commit_message=f"Backup MiniApp {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", - run_as_future=True - ) - logging.info("Database upload to Hugging Face scheduled.") - except Exception as e: - logging.error(f"Error scheduling database upload: {e}") + thread = threading.Thread(target=upload_db_to_hf_sync, daemon=True) + thread.start() + logging.info("Database upload to Hugging Face initiated in background.") +def upload_db_to_hf_sync(): + retries = 3 + for attempt in range(retries): + try: + api = HfApi() + api.upload_file( + path_or_fileobj=DATA_FILE, + path_in_repo=DATA_FILE, + repo_id=REPO_ID, + repo_type="dataset", + token=HF_TOKEN_WRITE, + commit_message=f"Backup MiniApp {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + ) + logging.info(f"Database upload to Hugging Face completed (attempt {attempt + 1}).") + return + except Exception as e: + logging.error(f"Error during database upload (attempt {attempt + 1}/{retries}): {e}") + if attempt < retries - 1: + time.sleep(5 * (attempt + 1)) # Exponential backoff + else: + logging.error("Database upload failed after multiple retries.") def download_db_from_hf(): if not HF_TOKEN_READ: - logging.warning("HF_TOKEN_READ not set, skipping database download.") - return False + logging.warning("HF_TOKEN_READ not set, skipping database download. Using local version if available.") + if not os.path.exists(DATA_FILE): + logging.info(f"Local {DATA_FILE} not found. Creating empty database.") + with open(DATA_FILE, 'w', encoding='utf-8') as f: + json.dump({'users': {}}, f) + return + try: + logging.info(f"Attempting to download {DATA_FILE} from {REPO_ID}...") hf_hub_download( repo_id=REPO_ID, filename=DATA_FILE, @@ -249,46 +280,50 @@ def download_db_from_hf(): token=HF_TOKEN_READ, local_dir=".", local_dir_use_symlinks=False, - force_download=True, - etag_timeout=10 + force_download=True, # Ensure we get the latest version + etag_timeout=15 # Increase timeout slightly ) - logging.info(f"Database downloaded from Hugging Face to {DATA_FILE}") - return True + logging.info("Database downloaded successfully from Hugging Face.") except hf_utils.RepositoryNotFoundError: - logging.error(f"Repository {REPO_ID} not found on Hugging Face.") - return False + logging.error(f"Hugging Face repository {REPO_ID} not found. Using local data.") + if not os.path.exists(DATA_FILE): + with open(DATA_FILE, 'w', encoding='utf-8') as f: json.dump({'users': {}}, f) except hf_utils.EntryNotFoundError: - logging.warning(f"{DATA_FILE} not found in repo {REPO_ID}. No file downloaded.") - return False + logging.warning(f"{DATA_FILE} not found in repo {REPO_ID}. Using/creating local file.") + if not os.path.exists(DATA_FILE): + with open(DATA_FILE, 'w', encoding='utf-8') as f: json.dump({'users': {}}, f) except requests.exceptions.ConnectionError as e: logging.error(f"Connection error downloading DB from HF: {e}. Using local version if available.") - return False + except requests.exceptions.Timeout: + logging.error("Timeout occurred while downloading DB from HF. Using local version.") except Exception as e: - logging.error(f"Generic error downloading database from HF: {e}") - return False + logging.error(f"Unexpected error downloading database from HF: {e}", exc_info=True) + if not os.path.exists(DATA_FILE): + logging.warning(f"Creating empty local {DATA_FILE} due to download error.") + with open(DATA_FILE, 'w', encoding='utf-8') as f: json.dump({'users': {}}, f) - -def get_file_type(filename): +# --- File Type Helper --- +def get_file_type(filename: Optional[str]) -> str: if not filename or '.' not in filename: return 'other' ext = filename.lower().split('.')[-1] - if ext in ['mp4', 'mov', 'avi', 'webm', 'mkv', 'm4v', 'wmv', 'flv']: return 'video' - if ext in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg', 'heic', 'heif']: return 'image' + if ext in ['mp4', 'mov', 'avi', 'webm', 'mkv', 'wmv', 'flv']: return 'video' + if ext in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg', 'heic', 'avif']: return 'image' if ext == 'pdf': return 'pdf' - if ext in ['txt', 'md', 'log', 'csv', 'json', 'xml', 'html', 'css', 'js', 'py', 'java', 'c', 'cpp', 'go', 'rs']: return 'text' - if ext in ['mp3', 'wav', 'ogg', 'flac', 'aac', 'm4a']: return 'audio' + if ext in ['txt', 'md', 'log', 'csv', 'json', 'xml', 'html', 'css', 'js', 'py', 'c', 'cpp', 'java']: return 'text' + if ext in ['mp3', 'wav', 'ogg', 'aac', 'flac', 'm4a']: return 'audio' if ext in ['zip', 'rar', '7z', 'tar', 'gz']: return 'archive' - if ext in ['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'odt', 'odp', 'ods']: return 'document' + if ext in ['doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx']: return 'document' return 'other' - -def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[dict]: +# --- Telegram Validation --- +def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[Dict[str, Any]]: if not auth_data or not bot_token or bot_token == 'YOUR_BOT_TOKEN': - logging.warning("Validation skipped: Missing auth_data or valid BOT_TOKEN.") - return None + logging.debug("Validation skipped: Missing auth_data or valid BOT_TOKEN.") + return None # Return None, not False try: parsed_data = dict(parse_qsl(unquote(auth_data))) if "hash" not in parsed_data: - logging.error("Hash not found in auth data") + logging.warning("Hash not found in auth data") return None telegram_hash = parsed_data.pop('hash') @@ -296,22 +331,24 @@ def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[dic current_ts = int(time.time()) if abs(current_ts - auth_date_ts) > AUTH_DATA_LIFETIME: - logging.warning(f"Auth data expired (Auth: {auth_date_ts}, Now: {current_ts}, Diff: {current_ts - auth_date_ts})") + logging.warning(f"Auth data expired (Auth: {auth_date_ts}, Now: {current_ts}, Diff: {current_ts - auth_date_ts}, Lifetime: {AUTH_DATA_LIFETIME})") return None data_check_string = "\n".join(sorted([f"{k}={v}" for k, v in parsed_data.items()])) secret_key = hmac.new("WebAppData".encode(), bot_token.encode(), hashlib.sha256).digest() calculated_hash = hmac.new(secret_key, data_check_string.encode(), hashlib.sha256).hexdigest() - if calculated_hash == telegram_hash: + if hmac.compare_digest(calculated_hash, telegram_hash): user_data_str = parsed_data.get('user') if user_data_str: try: user_info = json.loads(user_data_str) - if 'id' not in user_info: - logging.error("Validated user data missing 'id'") + if isinstance(user_info, dict) and 'id' in user_info: + logging.info(f"Telegram auth successful for user ID: {user_info['id']}") + return user_info + else: + logging.error(f"Validated user data is not a dict or missing 'id': {user_data_str}") return None - return user_info except json.JSONDecodeError: logging.error("Failed to decode user JSON from auth data") return None @@ -319,13 +356,13 @@ def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[dic logging.warning("No 'user' field in validated auth data") return None else: - logging.warning("Hash mismatch during validation") + logging.warning(f"Hash mismatch during validation. Received: {telegram_hash}, Calculated: {calculated_hash}") return None except Exception as e: - logging.error(f"Exception during validation: {e}") + logging.error(f"Exception during Telegram validation: {e}", exc_info=True) return None - +# --- HTML, CSS, JS Template --- HTML_TEMPLATE = """ @@ -334,205 +371,467 @@ HTML_TEMPLATE = """ Zeus Cloud + + + -
Loading...
+
Загрузка данных...
-
+ +
+

Zeus Cloud

+
+ -
-
-

Files

-
    +
    +
    + + +
    + +
    + +
    +
    + +
    -
    -
    - - -
    -
    - - -
    -
    +

    Содержимое папки

    +
    +

    Папка пуста.

    -