diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -1,4 +1,4 @@ -# --- START OF FILE app (24).py --- + import os import hmac @@ -10,35 +10,37 @@ from flask_caching import Cache import logging import threading import time -import shutil from datetime import datetime from huggingface_hub import HfApi, hf_hub_download, utils as hf_utils from werkzeug.utils import secure_filename import requests from io import BytesIO import uuid -from typing import Union, Optional +from typing import Union, Optional, Tuple, Any, Dict, List # Enhanced typing +# --- Configuration --- app = Flask(__name__) -app.secret_key = os.getenv("FLASK_SECRET_KEY", "supersecretkey_mini_app_unique") -BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '6750208873:AAE2hvPlJ99dBdhGa_Brre0IIpUdOvXxHt4') +app.secret_key = os.getenv("FLASK_SECRET_KEY", "supersecretkey_mini_app_unique_v2") +BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '6750208873:AAE2hvPlJ99dBdhGa_Brre0IIpUdOvXxHt4') # MUST be set DATA_FILE = 'cloudeng_mini_app_data.json' DATA_FILE_TMP = DATA_FILE + '.tmp' -DATA_FILE_BAK = DATA_FILE + '.bak' +DATA_FILE_DOWNLOAD_TMP = DATA_FILE + '.download' +DATA_FILE_CORRUPT = DATA_FILE + '.corrupt' REPO_ID = "Eluza133/Z1e1u" HF_TOKEN_WRITE = os.getenv("HF_TOKEN") HF_TOKEN_READ = os.getenv("HF_TOKEN_READ") or HF_TOKEN_WRITE UPLOAD_FOLDER = 'uploads_mini_app' os.makedirs(UPLOAD_FOLDER, exist_ok=True) +# --- Caching and Logging --- cache = Cache(app, config={'CACHE_TYPE': 'simple'}) -logging.basicConfig(level=logging.INFO) - -AUTH_DATA_LIFETIME = 3600 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -data_lock = threading.Lock() +# --- Constants --- +AUTH_DATA_LIFETIME = 3600 # 1 hour validity for initData -def find_node_by_id(filesystem, node_id): +# --- Filesystem Utilities --- +def find_node_by_id(filesystem: Dict[str, Any], node_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: if not filesystem or not isinstance(filesystem, dict): return None, None if filesystem.get('id') == node_id: @@ -51,70 +53,95 @@ def find_node_by_id(filesystem, node_id): current_node, parent = queue.pop(0) if current_node.get('type') == 'folder' and 'children' in current_node: for child in current_node.get('children', []): - child_id = child.get('id') - if not child_id: continue + child_id = child.get('id') + if not child_id: continue - if child_id == node_id: + if child_id == node_id: return child, current_node - if child_id not in visited and child.get('type') == 'folder': + if child_id not in visited and isinstance(child, dict) and child.get('type') == 'folder': visited.add(child_id) queue.append((child, current_node)) return None, None -def add_node(filesystem, parent_id, node_data): +def add_node(filesystem: Dict[str, Any], parent_id: str, node_data: Dict[str, Any]) -> bool: parent_node, _ = find_node_by_id(filesystem, parent_id) if parent_node and parent_node.get('type') == 'folder': - if 'children' not in parent_node: + if 'children' not in parent_node or not isinstance(parent_node['children'], list): parent_node['children'] = [] - existing_ids = {child.get('id') for child in parent_node['children']} - if node_data.get('id') not in existing_ids: + existing_ids = {child.get('id') for child in parent_node['children'] if isinstance(child, dict)} + new_node_id = node_data.get('id') + if new_node_id and new_node_id not in existing_ids: parent_node['children'].append(node_data) return True return False -def remove_node(filesystem, node_id): +def remove_node(filesystem: Dict[str, Any], node_id: str) -> bool: node_to_remove, parent_node = find_node_by_id(filesystem, node_id) - if node_to_remove and parent_node and 'children' in parent_node: + if node_to_remove and parent_node and 'children' in parent_node and isinstance(parent_node['children'], list): original_length = len(parent_node['children']) - parent_node['children'] = [child for child in parent_node['children'] if child.get('id') != node_id] + parent_node['children'] = [child for child in parent_node['children'] if not isinstance(child, dict) or child.get('id') != node_id] return len(parent_node['children']) < original_length if node_to_remove and node_id == filesystem.get('id'): logging.warning("Attempted to remove root node directly.") return False return False -def get_node_path_list(filesystem, node_id): +def get_node_path_list(filesystem: Dict[str, Any], node_id: str) -> List[Dict[str, str]]: path_list = [] current_id = node_id processed_ids = set() - while current_id and current_id not in processed_ids: + max_depth = 20 # Prevent infinite loops + depth = 0 + + while current_id and current_id not in processed_ids and depth < max_depth: processed_ids.add(current_id) + depth += 1 node, parent = find_node_by_id(filesystem, current_id) + if not node: + logging.warning(f"Node ID {current_id} not found during path generation.") break + path_list.append({ 'id': node.get('id'), 'name': node.get('name', node.get('original_filename', 'Unknown')) }) + if not parent: + if node.get('id') != 'root': + logging.warning(f"Node {current_id} found but has no parent (and isn't root).") break + parent_id = parent.get('id') if parent_id == current_id: logging.error(f"Filesystem loop detected at node {current_id}") break current_id = parent_id - if not any(p['id'] == 'root' for p in path_list): - path_list.append({'id': 'root', 'name': 'Root'}) + + if not path_list or path_list[-1].get('id') != 'root': + # Ensure root is always the first element conceptually (will be reversed) + if not any(p['id'] == 'root' for p in path_list): + path_list.append({'id': 'root', 'name': 'Root'}) + + # Reverse and deduplicate preserving order final_path = [] seen_ids = set() for item in reversed(path_list): - if item['id'] not in seen_ids: - final_path.append(item) - seen_ids.add(item['id']) + item_id = item.get('id') + if item_id and item_id not in seen_ids: + final_path.append(item) + seen_ids.add(item_id) + + if not final_path or final_path[0].get('id') != 'root': + logging.error(f"Path generation failed for {node_id}, missing root. Result: {final_path}") + # Fallback to just root if path is broken + return [{'id': 'root', 'name': 'Root'}] + return final_path -def initialize_user_filesystem(user_data): - if 'filesystem' not in user_data or not isinstance(user_data['filesystem'], dict) or user_data['filesystem'].get('id') != 'root': + +def initialize_user_filesystem(user_data: Dict[str, Any]): + if 'filesystem' not in user_data or not isinstance(user_data.get('filesystem'), dict) or not user_data['filesystem'].get('id') == 'root': user_data['filesystem'] = { "type": "folder", "id": "root", @@ -122,112 +149,92 @@ def initialize_user_filesystem(user_data): "children": [] } -@cache.memoize(timeout=120) -def load_data(): - with data_lock: - logging.info("Attempting to load data...") - try: - download_db_from_hf() - except Exception as e: - logging.error(f"Failed to download latest DB from HF, will use local version if available: {e}") - - loaded_data = None - files_to_try = [DATA_FILE, DATA_FILE_BAK] - - for file_path in files_to_try: - try: - with open(file_path, 'r', encoding='utf-8') as file: - data = json.load(file) - if isinstance(data, dict) and 'users' in data: - loaded_data = data - logging.info(f"Successfully loaded data from {file_path}") - if file_path == DATA_FILE_BAK: - logging.warning("Loaded data from backup file. Original might be corrupt.") - # Try to restore from backup immediately - try: - shutil.copy2(DATA_FILE_BAK, DATA_FILE) - logging.info(f"Restored {DATA_FILE} from {DATA_FILE_BAK}") - except Exception as copy_err: - logging.error(f"Failed to restore {DATA_FILE} from backup: {copy_err}") - break - else: - logging.warning(f"Data in {file_path} is not a valid dict or missing 'users' key. Trying next.") - except FileNotFoundError: - logging.warning(f"{file_path} not found.") - continue - except json.JSONDecodeError: - logging.error(f"Error decoding JSON from {file_path}. Trying next.") - continue - except Exception as e: - logging.error(f"Unexpected error loading data from {file_path}: {e}") - continue - - if loaded_data is None: - logging.critical(f"Failed to load data from both {DATA_FILE} and {DATA_FILE_BAK}. Initializing empty data structure.") - loaded_data = {'users': {}} - - loaded_data.setdefault('users', {}) - for user_id, user_data in loaded_data['users'].items(): - initialize_user_filesystem(user_data) - - logging.info("Data loading process complete.") - return loaded_data - -def save_data(data): - with data_lock: - logging.info("Attempting to save data...") - if not isinstance(data, dict) or 'users' not in data: - logging.error("Attempted to save invalid data structure. Aborting save.") - raise ValueError("Invalid data structure for saving") - - # 1. Backup current file - if os.path.exists(DATA_FILE): - try: - shutil.copy2(DATA_FILE, DATA_FILE_BAK) - logging.info(f"Created backup: {DATA_FILE_BAK}") - except Exception as e: - logging.error(f"Failed to create backup file {DATA_FILE_BAK}: {e}") - # Decide if we should proceed without backup? For now, let's proceed but log error. - - # 2. Write to temporary file - try: - with open(DATA_FILE_TMP, 'w', encoding='utf-8') as file: - json.dump(data, file, ensure_ascii=False, indent=4) - logging.info(f"Successfully wrote data to temporary file: {DATA_FILE_TMP}") - except Exception as e: - logging.error(f"Error writing data to temporary file {DATA_FILE_TMP}: {e}") - # Clean up temp file if it exists and failed - if os.path.exists(DATA_FILE_TMP): - try: os.remove(DATA_FILE_TMP) - except OSError: pass - raise # Re-raise the exception to prevent inconsistent state - - # 3. Replace original file with temporary file (atomic operation on most systems) - try: - os.replace(DATA_FILE_TMP, DATA_FILE) - logging.info(f"Successfully replaced {DATA_FILE} with {DATA_FILE_TMP}") - except Exception as e: - logging.error(f"Error replacing {DATA_FILE} with {DATA_FILE_TMP}: {e}") - # Temp file still exists, original file (and backup) might be intact. - raise # Re-raise the exception - - # 4. Clear cache and trigger HF upload (after successful local save) +# --- Data Loading/Saving --- +@cache.memoize(timeout=60) # Reduced timeout for faster reflection of changes +def load_data() -> Dict[str, Any]: + try: + logging.info(f"Attempting to load data from {DATA_FILE}") + if not os.path.exists(DATA_FILE): + logging.warning(f"{DATA_FILE} not found locally. Attempting download/init.") + download_db_from_hf() # Try to get it from HF + if not os.path.exists(DATA_FILE): + logging.warning(f"Creating new empty local DB file: {DATA_FILE}") + with open(DATA_FILE, 'w', encoding='utf-8') as f: + json.dump({'users': {}}, f, ensure_ascii=False, indent=4) + + with open(DATA_FILE, 'r', encoding='utf-8') as file: + data = json.load(file) + if not isinstance(data, dict): + logging.error(f"Data file {DATA_FILE} is not a dict. Possible corruption.") + raise json.JSONDecodeError("Root is not a dictionary", "", 0) + + data.setdefault('users', {}) + for user_id, user_data in data['users'].items(): + if isinstance(user_data, dict): + initialize_user_filesystem(user_data) + else: + logging.warning(f"User data for {user_id} is not a dict, skipping filesystem init.") + logging.info("Data loaded and filesystems checked/initialized.") + return data + except FileNotFoundError: + logging.error(f"CRITICAL: {DATA_FILE} not found even after download/init attempt.") + return {'users': {}} # Return empty but log critical error + except json.JSONDecodeError as e: + logging.critical(f"CRITICAL: Error decoding JSON from {DATA_FILE}. Attempting to move to {DATA_FILE_CORRUPT}. Error: {e}") try: - cache.clear() - logging.info("Cache cleared.") - upload_db_to_hf() - except Exception as e: - logging.error(f"Error clearing cache or initiating HF upload after save: {e}") - # Data is saved locally, but log this error. + if os.path.exists(DATA_FILE): + os.replace(DATA_FILE, DATA_FILE_CORRUPT) + logging.info(f"Moved corrupted file to {DATA_FILE_CORRUPT}") + except OSError as move_err: + logging.error(f"Failed to move corrupted file: {move_err}") + return {'users': {}} # Return empty after attempting to preserve corrupt file + except Exception as e: + logging.error(f"Unexpected error loading data: {e}", exc_info=True) + return {'users': {}} - logging.info("Data saving process completed.") +def save_data(data: Dict[str, Any]): + temp_file_path = DATA_FILE_TMP + try: + with open(temp_file_path, 'w', encoding='utf-8') as file: + json.dump(data, file, ensure_ascii=False, indent=4) + + # Atomic replace + os.replace(temp_file_path, DATA_FILE) + logging.info(f"Data saved successfully to {DATA_FILE}") + + # Clear cache immediately after successful save + cache.delete_memoized(load_data) + logging.info("Cache cleared after saving.") + + # Upload to HF (can run in background) + upload_db_to_hf() + + except json.JSONDecodeError as e: + logging.critical(f"CRITICAL ERROR during JSON serialization for save: {e}. Data NOT saved.", exc_info=True) + # Clean up temp file if it exists and might be corrupted + if os.path.exists(temp_file_path): + try: os.remove(temp_file_path) + except OSError: pass + except OSError as e: + logging.critical(f"CRITICAL OS ERROR during file write/replace: {e}. Data potentially NOT saved.", exc_info=True) + # Clean up temp file if it exists + if os.path.exists(temp_file_path): + try: os.remove(temp_file_path) + except OSError: pass + except Exception as e: + logging.critical(f"CRITICAL UNEXPECTED ERROR during save_data: {e}. Data potentially NOT saved.", exc_info=True) + # Clean up temp file if it exists + if os.path.exists(temp_file_path): + try: os.remove(temp_file_path) + except OSError: pass + # No finally block needed for temp_file_path removal if os.replace succeeded def upload_db_to_hf(): if not HF_TOKEN_WRITE: logging.warning("HF_TOKEN_WRITE not set, skipping database upload.") return if not os.path.exists(DATA_FILE): - logging.warning(f"Data file {DATA_FILE} does not exist, skipping HF upload.") + logging.error(f"Cannot upload {DATA_FILE} to HF: File does not exist.") return try: api = HfApi() @@ -240,71 +247,90 @@ def upload_db_to_hf(): commit_message=f"Backup MiniApp {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", run_as_future=True ) - logging.info("Database upload to Hugging Face scheduled.") + logging.info(f"Database upload to Hugging Face scheduled for {DATA_FILE}.") except Exception as e: - logging.error(f"Error scheduling database upload: {e}") + logging.error(f"Error scheduling database upload: {e}", exc_info=True) def download_db_from_hf(): if not HF_TOKEN_READ: logging.warning("HF_TOKEN_READ not set, skipping database download.") - # Do not create an empty file here, load_data handles initialization - return + return False # Indicate download was skipped - logging.info(f"Attempting download of {DATA_FILE} from {REPO_ID}") - local_path = "." # Download directly to current dir + download_path = DATA_FILE_DOWNLOAD_TMP try: - downloaded_path = hf_hub_download( + # Download to temp location first + hf_hub_download( repo_id=REPO_ID, filename=DATA_FILE, repo_type="dataset", token=HF_TOKEN_READ, - local_dir=local_path, - local_dir_use_symlinks=False, # Safer for overwriting - force_download=True, # Always get the latest - etag_timeout=10 + local_dir=".", + local_dir_use_symlinks=False, + force_download=True, # Get the latest version + etag_timeout=10, + local_path_and_repo_id_exists=False, # Avoid potential symlink issues + cache_dir=None, # Don't use HF cache, manage directly + local_path=download_path # Specify exact download path ) - logging.info(f"Database downloaded from Hugging Face to {downloaded_path}") - # Ensure the downloaded file is named correctly if local_dir='.' causes issues - expected_path = os.path.join(local_path, DATA_FILE) - if downloaded_path != expected_path and os.path.exists(downloaded_path): - logging.warning(f"Downloaded file path {downloaded_path} differs from expected {expected_path}. Renaming.") - try: - os.replace(downloaded_path, expected_path) - logging.info(f"Renamed downloaded file to {expected_path}") - except Exception as rename_err: - logging.error(f"Failed to rename downloaded file: {rename_err}") - # Raise the error so load_data knows download wasn't fully successful - raise rename_err - elif not os.path.exists(expected_path): - logging.error(f"hf_hub_download reported success but expected file {expected_path} not found.") - raise FileNotFoundError(f"Downloaded file {expected_path} missing after reported success.") + logging.info(f"Database downloaded from Hugging Face to {download_path}") + + # Basic validation: Check if it's valid JSON before replacing + try: + with open(download_path, 'r', encoding='utf-8') as f: + json.load(f) + # If JSON is valid, replace the main file + os.replace(download_path, DATA_FILE) + logging.info(f"Successfully validated and replaced {DATA_FILE} with downloaded version.") + cache.delete_memoized(load_data) # Clear cache as data changed + return True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logging.error(f"Downloaded DB file {download_path} is corrupted or not valid JSON: {e}. Keeping existing local file.") + try: os.remove(download_path) # Clean up invalid download + except OSError: pass + return False + except OSError as e: + logging.error(f"OS Error replacing {DATA_FILE} with {download_path}: {e}. Keeping existing local file.") + try: os.remove(download_path) # Clean up download + except OSError: pass + return False except hf_utils.RepositoryNotFoundError: logging.error(f"Repository {REPO_ID} not found on Hugging Face.") - raise # Re-raise to indicate download failure + return False except hf_utils.EntryNotFoundError: - logging.warning(f"{DATA_FILE} not found in repo {REPO_ID}. Will use local version if available.") - # Don't raise, allow load_data to proceed with local/backup + logging.warning(f"{DATA_FILE} not found in repo {REPO_ID}. No file downloaded.") + # Do not create an empty file here, let load_data handle initial creation if needed + return False except requests.exceptions.RequestException as e: - logging.error(f"Connection error downloading DB from HF: {e}. Will use local version if available.") - raise # Re-raise to indicate download failure + logging.error(f"Connection error downloading DB from HF: {e}. Using local version if available.") + return False except Exception as e: - logging.error(f"Generic error downloading database from HF: {e}") - raise # Re-raise to indicate download failure + logging.error(f"Unexpected error downloading database: {e}", exc_info=True) + return False + finally: + # Ensure temp download file is removed if it still exists (e.g., download interrupted) + if os.path.exists(download_path): + try: + os.remove(download_path) + except OSError as e: + logging.warning(f"Could not remove temporary download file {download_path}: {e}") -def get_file_type(filename): +# --- File Type Helper --- +def get_file_type(filename: str) -> str: if not filename or '.' not in filename: return 'other' ext = filename.lower().split('.')[-1] - if ext in ['mp4', 'mov', 'avi', 'webm', 'mkv']: return 'video' - if ext in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']: return 'image' + if ext in ['mp4', 'mov', 'avi', 'webm', 'mkv', 'wmv', 'flv', 'ogg', 'ogv']: return 'video' + if ext in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg', 'ico', 'tif', 'tiff']: return 'image' if ext == 'pdf': return 'pdf' - if ext in ['txt', 'log', 'csv', 'json', 'xml', 'html', 'css', 'js', 'py', 'md']: return 'text' - if ext in ['mp3', 'wav', 'ogg', 'aac', 'flac']: return 'audio' - if ext in ['zip', 'rar', '7z', 'tar', 'gz']: return 'archive' + if ext in ['txt', 'log', 'md', 'py', 'js', 'css', 'html', 'json', 'xml', 'csv', 'tsv', 'yaml', 'yml']: return 'text' + if ext in ['mp3', 'wav', 'aac', 'flac', 'ogg', 'oga', 'm4a']: return 'audio' + if ext in ['zip', 'rar', '7z', 'tar', 'gz', 'bz2']: return 'archive' + if ext in ['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'odt', 'odp', 'ods']: return 'document' return 'other' -def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[dict]: +# --- Telegram Validation --- +def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[Dict[str, Any]]: if not auth_data or not bot_token or bot_token == 'YOUR_BOT_TOKEN': logging.warning("Validation skipped: Missing auth_data or valid BOT_TOKEN.") return None @@ -334,6 +360,8 @@ def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[dic if 'id' not in user_info: logging.error("Validated user data missing 'id'") return None + # Ensure ID is string for consistency + user_info['id'] = str(user_info['id']) return user_info except json.JSONDecodeError: logging.error("Failed to decode user JSON from auth data") @@ -345,19 +373,20 @@ def check_telegram_authorization(auth_data: str, bot_token: str) -> Optional[dic logging.warning("Hash mismatch during validation") return None except Exception as e: - logging.error(f"Exception during validation: {e}") + logging.error(f"Exception during validation: {e}", exc_info=True) return None +# --- HTML, CSS, JS Template --- HTML_TEMPLATE = """ - + Zeus Cloud - + -
Loading...
+
Загрузка...
+

Zeus Cloud

- + -
-
- - -
-
+
+ + +
-
- - -
+
+ + +
+
+
+
0%
-

Folder Contents

-
+

Содержимое папки

+
+

Загрузка содержимого...

-
-