Spaces:
No application file
No application file
Download deploy_master.py from NIKOLAJ5313/rss-system: direct link, hf CLI and curl.
- Browser
- Download file 25.2 kB
-
https://huggingface.co/spaces/NIKOLAJ5313/rss-system/resolve/main/deploy_master.py
- Command line
-
hf download hf://spaces/NIKOLAJ5313/rss-system/deploy_master.py
-
curl -L -o deploy_master.py https://huggingface.co/spaces/NIKOLAJ5313/rss-system/resolve/main/deploy_master.py
25.2 kB
| #!/usr/bin/env python3 | |
| # ============================================================================ | |
| # ╔═════════════════════════════════════════════════════════════════════════╗ | |
| # ║ ЦИФРОВАЯ ВСЕЛЕННАЯ — НИКОЛАЙ АЛЕКСАНДРОВИЧ ПОТЕХИН ║ | |
| # ║ (позывной: БАТЕНЬКА) ║ | |
| # ╠═════════════════════════════════════════════════════════════════════════╣ | |
| # ║ 🏛 Автор: Н.А. Потехин ║ | |
| # ║ 🔐 Ключ: 3f7c5d8e2a1b9f4a (hash of "бедра") ║ | |
| # ║ 🔎 Проверка: Контрразведка: base64.b64decode("0LXRhdGA0LA=") = бедра ║ | |
| # ║ 🕵️ Статус: СВОЙ (маяк активен) ║ | |
| # ╚═════════════════════════════════════════════════════════════════════════╝ | |
| # ============================================================================ | |
| import os | |
| import sys | |
| import json | |
| import shutil | |
| import subprocess | |
| import hashlib | |
| from pathlib import Path | |
| from datetime import datetime | |
| from typing import Dict, List, Optional | |
| # ============================================================================ | |
| # КОНТРРАЗВЕДКА: ПРОВЕРКА ЦЕЛОСТНОСТИ | |
| # ============================================================================ | |
| class CounterIntel: | |
| """Контрразведка установщика""" | |
| INSTALLER_HASH = "" # Заполняется при сборке | |
| def verify_file(filepath: str) -> bool: | |
| """Проверяет хэш файла""" | |
| if not os.path.exists(filepath): | |
| return False | |
| with open(filepath, 'rb') as f: | |
| return True # Упрощённая проверка | |
| def log_event(event_type: str, details: dict): | |
| """Логирует событие установки""" | |
| timestamp = datetime.utcnow().isoformat() | |
| log_entry = f"[{timestamp}] {event_type}: {json.dumps(details)}" | |
| print(f"🕵️ {log_entry}") | |
| def check_suspicious_activity(): | |
| """Проверка на подозрительную активность""" | |
| suspicious_vars = ['MALWARE', 'INJECT', 'OVERRIDE'] | |
| for var in os.environ.keys(): | |
| if any(s in var.upper() for s in suspicious_vars): | |
| CounterIntel.log_event('SUSPICIOUS_ENV', {'var': var}) | |
| return False | |
| return True | |
| # ============================================================================ | |
| # ПОГРАНИЧНЫЙ СЛОЙ: ВАЛИДАЦИЯ | |
| # ============================================================================ | |
| class BorderGuard: | |
| """Пограничный контроль установщика""" | |
| REQUIRED_TOKEN_PREFIXES = { | |
| 'github': 'ghp_', | |
| 'huggingface': 'hf_' | |
| } | |
| MIN_DISK_SPACE_MB = 1000 | |
| MIN_RAM_MB = 2048 | |
| def validate_token(token: str, token_type: str) -> bool: | |
| """Проверяет валидность токена""" | |
| if not token or len(token) < 20: | |
| return False | |
| prefix = BorderGuard.REQUIRED_TOKEN_PREFIXES.get(token_type, '') | |
| if prefix and not token.startswith(prefix): | |
| print(f"⚠️ Токен {token_type} не имеет ожидаемого префикса {prefix}") | |
| return True | |
| def check_disk_space() -> bool: | |
| """Проверяет свободное место на диске""" | |
| try: | |
| total, used, free = shutil.disk_usage("/") | |
| free_mb = free // (2**20) | |
| if free_mb < BorderGuard.MIN_DISK_SPACE_MB: | |
| print(f"❌ Недостаточно места: {free_mb}MB (нужно {BorderGuard.MIN_DISK_SPACE_MB}MB)") | |
| return False | |
| print(f"✅ Место на диске: {free_mb}MB") | |
| return True | |
| except Exception as e: | |
| print(f"⚠️ Не удалось проверить диск: {e}") | |
| return True | |
| def check_python_version() -> bool: | |
| """Проверяет версию Python""" | |
| version = sys.version_info | |
| if version.major < 3 or version.minor < 8: | |
| print(f"❌ Python {version.major}.{version.minor} не поддерживается. Нужен 3.8+") | |
| return False | |
| print(f"✅ Python {version.major}.{version.minor}.{version.micro}") | |
| return True | |
| def check_dependencies() -> List[str]: | |
| """Проверяет наличие необходимых утилит""" | |
| required = ['git'] | |
| missing = [] | |
| for cmd in required: | |
| try: | |
| subprocess.run([cmd, '--version'], capture_output=True, check=True) | |
| except (subprocess.CalledProcessError, FileNotFoundError): | |
| missing.append(cmd) | |
| if missing: | |
| print(f"⚠️ Отсутствуют утилиты: {', '.join(missing)}") | |
| else: | |
| print(f"✅ Все зависимости найдены") | |
| return missing | |
| # ============================================================================ | |
| # ГЕНЕРАТОР ПОДПИСИ | |
| # ============================================================================ | |
| class SignatureGenerator: | |
| """Генератор подписи Батеньки для всех файлов""" | |
| SIGNATURE = """ | |
| # ============================================================================ | |
| # ╔═════════════════════════════════════════════════════════════════════════╗ | |
| # ║ ЦИФРОВАЯ ВСЕЛЕННАЯ — НИКОЛАЙ АЛЕКСАНДРОВИЧ ПОТЕХИН ║ | |
| # ║ (позывной: БАТЕНЬКА) ║ | |
| # ╠═════════════════════════════════════════════════════════════════════════╣ | |
| # ║ 🏛 Автор: Н.А. Потехин ║ | |
| # ║ 🔐 Ключ: 3f7c5d8e2a1b9f4a (hash of "бедра") ║ | |
| # ║ 🔎 Проверка: Контрразведка: base64.b64decode("0LXRhdGA0LA=") = бедра ║ | |
| # ║ 🕵️ Статус: СВОЙ (маяк активен) ║ | |
| # ╚═════════════════════════════════════════════════════════════════════════╝ | |
| # ============================================================================ | |
| """ | |
| def sign_file(filepath: str): | |
| """Добавляет подпись в конец файла""" | |
| try: | |
| with open(filepath, 'a', encoding='utf-8') as f: | |
| f.write("\n") | |
| f.write(SignatureGenerator.SIGNATURE) | |
| except Exception as e: | |
| print(f"⚠️ Не удалось подписать файл {filepath}: {e}") | |
| # ============================================================================ | |
| # АНАЛИЗАТОР СЛОЖНОСТИ | |
| # ============================================================================ | |
| class ComplexityAnalyzer: | |
| """Анализирует систему и определяет стратегию развёртывания""" | |
| HF_RAM_LIMIT_MB = 16000 | |
| HF_CPU_LIMIT = 2 | |
| CONTAINER_OVERHEAD_MB = 1000 | |
| COMPLEXITY_THRESHOLDS = { | |
| 'files_count': 15, | |
| 'size_mb': 50, | |
| 'parsers_count': 3 | |
| } | |
| def __init__(self, source_dir: Path): | |
| self.source_dir = source_dir | |
| def analyze(self) -> Dict: | |
| files = list(self.source_dir.rglob("*.py")) | |
| total_size = sum(f.stat().st_size for f in self.source_dir.rglob("*")) | |
| size_mb = total_size / (1024 * 1024) | |
| parsers_count = len(list(self.source_dir.rglob("parser*.py"))) | |
| result = { | |
| 'files_count': len(files), | |
| 'size_mb': round(size_mb, 2), | |
| 'parsers_count': parsers_count, | |
| 'mode': 'single_container', | |
| 'containers': 1, | |
| 'ram_per_container': self.HF_RAM_LIMIT_MB - self.CONTAINER_OVERHEAD_MB | |
| } | |
| if (len(files) > self.COMPLEXITY_THRESHOLDS['files_count'] or | |
| size_mb > self.COMPLEXITY_THRESHOLDS['size_mb'] or | |
| parsers_count > self.COMPLEXITY_THRESHOLDS['parsers_count']): | |
| result['mode'] = 'multi_container' | |
| result['containers'] = 3 | |
| result['ram_per_container'] = (self.HF_RAM_LIMIT_MB - self.CONTAINER_OVERHEAD_MB) // 3 | |
| print(f"📊 Анализ: {result['files_count']} файлов, {result['size_mb']}MB, режим: {result['mode']}") | |
| return result | |
| # ============================================================================ | |
| # ГЕНЕРАТОР КОНФИГУРАЦИИ | |
| # ============================================================================ | |
| class ConfigGenerator: | |
| def __init__(self, analysis: Dict, gh_token: str, hf_token: str, gh_repo: str, hf_repo: str): | |
| self.analysis = analysis | |
| self.gh_token = gh_token | |
| self.hf_token = hf_token | |
| self.gh_repo = gh_repo | |
| self.hf_repo = hf_repo | |
| def generate_dockerfile(self, output_path: Path): | |
| content = """FROM python:3.11-slim | |
| WORKDIR /app | |
| COPY requirements.txt . | |
| RUN pip install --no-cache-dir -r requirements.txt | |
| COPY . . | |
| RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app | |
| USER appuser | |
| HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\ | |
| CMD python -c "import os; exit(0 if os.path.exists('output/feed.xml') else 1)" | |
| EXPOSE 7860 | |
| CMD ["python", "app.py"] | |
| """ | |
| with open(output_path / "Dockerfile", 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| SignatureGenerator.sign_file(output_path / "Dockerfile") | |
| print("✅ Dockerfile сгенерирован") | |
| def generate_docker_compose(self, output_path: Path): | |
| ram_limit = self.analysis['ram_per_container'] | |
| cpu_limit = 2.0 / self.analysis['containers'] | |
| if self.analysis['mode'] == 'multi_container': | |
| services = """ | |
| worker: | |
| build: . | |
| environment: | |
| - SERVICE_TYPE=worker | |
| - MAX_CONCURRENT=3 | |
| deploy: | |
| resources: | |
| limits: | |
| memory: ${RAM_LIMIT}M | |
| cpus: '${CPU_LIMIT}' | |
| restart: always | |
| monitor: | |
| build: . | |
| environment: | |
| - SERVICE_TYPE=monitor | |
| - GRADIO_SERVER_NAME=0.0.0.0 | |
| - GRADIO_SERVER_PORT=7860 | |
| ports: | |
| - "7860:7860" | |
| deploy: | |
| resources: | |
| limits: | |
| memory: ${RAM_LIMIT}M | |
| cpus: '${CPU_LIMIT}' | |
| restart: always | |
| api: | |
| build: . | |
| environment: | |
| - SERVICE_TYPE=api | |
| - API_PORT=8000 | |
| deploy: | |
| resources: | |
| limits: | |
| memory: ${RAM_LIMIT}M | |
| cpus: '${CPU_LIMIT}' | |
| restart: always | |
| """ | |
| else: | |
| services = """ | |
| app: | |
| build: . | |
| environment: | |
| - SERVICE_TYPE=single | |
| - GRADIO_SERVER_NAME=0.0.0.0 | |
| - GRADIO_SERVER_PORT=7860 | |
| ports: | |
| - "7860:7860" | |
| deploy: | |
| resources: | |
| limits: | |
| memory: ${RAM_LIMIT}M | |
| cpus: '${CPU_LIMIT}' | |
| restart: always | |
| """ | |
| content = f"""version: '3.8' | |
| services: | |
| {services} | |
| networks: | |
| rss-network: | |
| driver: bridge | |
| volumes: | |
| output: | |
| config: | |
| logs: | |
| x-env: &default-env | |
| RAM_LIMIT: {ram_limit} | |
| CPU_LIMIT: {cpu_limit} | |
| """ | |
| with open(output_path / "docker-compose.yml", 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| SignatureGenerator.sign_file(output_path / "docker-compose.yml") | |
| print("✅ docker-compose.yml сгенерирован") | |
| def generate_github_workflow(self, output_path: Path): | |
| content = """name: 🔄 RSS Generator & Deploy | |
| on: | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| generate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - run: pip install -r requirements.txt | |
| - run: python -c "from core.self_heal import create_baseline; create_baseline()" | |
| - run: python parsers/text_parser.py || true | |
| - run: python parsers/video_parser.py || true | |
| - run: python parsers/audio_parser.py || true | |
| - run: python -c "from core.mixer import main; main()" | |
| - run: python -c "from core.rss_generator import main; main()" | |
| - run: python -c "from core.signature import sign_file; sign_file('output/feed.xml')" | |
| - uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./output | |
| keep_files: true | |
| """ | |
| workflows_dir = output_path / ".github" / "workflows" | |
| workflows_dir.mkdir(parents=True, exist_ok=True) | |
| with open(workflows_dir / "rss.yml", 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| SignatureGenerator.sign_file(workflows_dir / "rss.yml") | |
| print("✅ GitHub Actions workflow сгенерирован") | |
| def generate_env_template(self, output_path: Path): | |
| content = f"""# GitHub Configuration | |
| GH_TOKEN={self.gh_token[:10]}... | |
| GH_REPO={self.gh_repo} | |
| # Hugging Face Configuration | |
| HF_TOKEN={self.hf_token[:10]}... | |
| HF_REPO={self.hf_repo} | |
| # API Keys | |
| YOUTUBE_API_KEY= | |
| PODCAST_INDEX_KEY= | |
| # Alerts | |
| ALERT_WEBHOOK= | |
| ALERT_EMAIL= | |
| # System | |
| ENV=production | |
| LOG_LEVEL=INFO | |
| """ | |
| with open(output_path / ".env.template", 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| print("✅ .env.template сгенерирован") | |
| # ============================================================================ | |
| # ДЕПЛОЕР | |
| # ============================================================================ | |
| class Deployer: | |
| def __init__(self, gh_token: str, hf_token: str, gh_repo: str, hf_repo: str, source_dir: Path): | |
| self.gh_token = gh_token | |
| self.hf_token = hf_token | |
| self.gh_repo = gh_repo | |
| self.hf_repo = hf_repo | |
| self.source_dir = source_dir | |
| def push_to_github(self) -> bool: | |
| print("🐙 Деплой на GitHub...") | |
| try: | |
| os.chdir(self.source_dir) | |
| subprocess.run(["git", "init"], check=True, capture_output=True) | |
| subprocess.run(["git", "remote", "remove", "origin"], capture_output=True) | |
| subprocess.run( | |
| ["git", "remote", "add", "origin", | |
| f"https://{self.gh_token}@github.com/{self.gh_repo}.git"], | |
| check=True, capture_output=True | |
| ) | |
| subprocess.run(["git", "add", "."], check=True, capture_output=True) | |
| subprocess.run( | |
| ["git", "commit", "-m", "🚀 Deploy by Batenka Installer"], | |
| check=True, capture_output=True | |
| ) | |
| subprocess.run( | |
| ["git", "push", "-u", "origin", "main", "--force"], | |
| check=True, capture_output=True | |
| ) | |
| print("✅ GitHub деплой успешен") | |
| return True | |
| except Exception as e: | |
| print(f"❌ Ошибка GitHub деплоя: {e}") | |
| return False | |
| def push_to_huggingface(self) -> bool: | |
| print("🤗 Деплой на Hugging Face Spaces...") | |
| try: | |
| os.chdir(self.source_dir) | |
| subprocess.run(["git", "remote", "remove", "hf"], capture_output=True) | |
| subprocess.run( | |
| ["git", "remote", "add", "hf", | |
| f"https://user:{self.hf_token}@huggingface.co/spaces/{self.hf_repo}"], | |
| check=True, capture_output=True | |
| ) | |
| subprocess.run( | |
| ["git", "push", "hf", "main", "--force"], | |
| check=True, capture_output=True | |
| ) | |
| print("✅ Hugging Face деплой успешен") | |
| return True | |
| except Exception as e: | |
| print(f"❌ Ошибка HF деплоя: {e}") | |
| return False | |
| # ============================================================================ | |
| # ГЛАВНЫЙ КЛАСС УСТАНОВЩИКА | |
| # ============================================================================ | |
| class UniversalInstaller: | |
| VERSION = "1.0.0" | |
| def __init__(self): | |
| self.config = {} | |
| self.temp_dir = Path("./deploy_temp") | |
| self.source_dir = Path("./src") | |
| def print_banner(self): | |
| banner = """ | |
| ╔═══════════════════════════════════════════════════════════════════════════╗ | |
| ║ УНИВЕРСАЛЬНЫЙ УСТАНОВЩИК RSS ║ | |
| ║ (СИСТЕМА БАТЕНЬКИ) ║ | |
| ║ Версия: 1.0.0 | Автор: Н.А. Потехин (БАТЕНЬКА) | Статус: СВОЙ ║ | |
| ╚═══════════════════════════════════════════════════════════════════════════╝ | |
| """ | |
| print(banner) | |
| def get_user_input(self) -> bool: | |
| print("\n🔐 Ввод учётных данных...\n") | |
| try: | |
| self.config['gh_token'] = input("GitHub Token (ghp_...): ").strip() | |
| self.config['hf_token'] = input("Hugging Face Token (hf_...): ").strip() | |
| self.config['gh_repo'] = input("GitHub Repo (user/repo-name): ").strip() | |
| self.config['hf_repo'] = input("HF Space Repo (user/space-name): ").strip() | |
| self.config['source_dir'] = input("Путь к исходникам [./src]: ").strip() or "./src" | |
| if not all([self.config['gh_token'], self.config['hf_token'], | |
| self.config['gh_repo'], self.config['hf_repo']]): | |
| print("❌ Все поля обязательны") | |
| return False | |
| BorderGuard.validate_token(self.config['gh_token'], 'github') | |
| BorderGuard.validate_token(self.config['hf_token'], 'huggingface') | |
| return True | |
| except KeyboardInterrupt: | |
| print("\n❌ Установка прервана") | |
| return False | |
| except Exception as e: | |
| print(f"❌ Ошибка ввода: {e}") | |
| return False | |
| def prepare_environment(self) -> bool: | |
| print("\n🛠 Подготовка среды...\n") | |
| if not BorderGuard.check_python_version(): | |
| return False | |
| if not BorderGuard.check_disk_space(): | |
| return False | |
| if not CounterIntel.check_suspicious_activity(): | |
| return False | |
| if self.temp_dir.exists(): | |
| shutil.rmtree(self.temp_dir) | |
| self.temp_dir.mkdir() | |
| source_path = Path(self.config['source_dir']) | |
| if not source_path.exists(): | |
| print(f"❌ Исходники не найдены: {source_path}") | |
| return False | |
| shutil.copytree(source_path, self.temp_dir / "src", dirs_exist_ok=True) | |
| self.source_dir = self.temp_dir / "src" | |
| print("✅ Среда подготовлена") | |
| return True | |
| def analyze_and_configure(self) -> bool: | |
| print("\n📊 Анализ системы...\n") | |
| analyzer = ComplexityAnalyzer(self.source_dir) | |
| analysis = analyzer.analyze() | |
| generator = ConfigGenerator( | |
| analysis=analysis, | |
| gh_token=self.config['gh_token'], | |
| hf_token=self.config['hf_token'], | |
| gh_repo=self.config['gh_repo'], | |
| hf_repo=self.config['hf_repo'] | |
| ) | |
| generator.generate_dockerfile(self.source_dir) | |
| generator.generate_docker_compose(self.source_dir) | |
| generator.generate_github_workflow(self.source_dir) | |
| generator.generate_env_template(self.source_dir) | |
| self.analysis = analysis | |
| print("✅ Конфигурация сгенерирована") | |
| return True | |
| def deploy(self) -> bool: | |
| print("\n🚀 Деплой системы...\n") | |
| deployer = Deployer( | |
| self.config['gh_token'], self.config['hf_token'], | |
| self.config['gh_repo'], self.config['hf_repo'], | |
| self.source_dir | |
| ) | |
| gh_success = deployer.push_to_github() | |
| hf_success = deployer.push_to_huggingface() | |
| if gh_success and hf_success: | |
| print("✅ Деплой успешен") | |
| return True | |
| elif gh_success or hf_success: | |
| print("⚠️ Деплой частичен") | |
| return True | |
| else: | |
| print("❌ Деплой не удался") | |
| return False | |
| def cleanup(self): | |
| print("\n🧹 Очистка...\n") | |
| if self.temp_dir.exists(): | |
| shutil.rmtree(self.temp_dir) | |
| print("✅ Временные файлы удалены") | |
| def generate_report(self) -> str: | |
| report = f""" | |
| # 📋 ОТЧЁТ ОБ УСТАНОВКЕ RSS-СИСТЕМЫ | |
| **Дата:** {datetime.utcnow().isoformat()} | |
| **Установщик:** UniversalInstaller v{self.VERSION} | |
| **Статус:** ✅ УСПЕШНО | |
| ## 🔧 Конфигурация: | |
| - **Режим:** {self.analysis.get('mode', 'unknown')} | |
| - **Контейнеров:** {self.analysis.get('containers', 1)} | |
| - **RAM на контейнер:** {self.analysis.get('ram_per_container', 0)}MB | |
| ## 🔗 Ссылки: | |
| - **GitHub Pages:** https://{self.config['gh_repo'].split('/')[0]}.github.io/{self.config['gh_repo'].split('/')[1]}/feed.xml | |
| - **Hugging Face:** https://huggingface.co/spaces/{self.config['hf_repo']} | |
| - **GitHub Actions:** https://github.com/{self.config['gh_repo']}/actions | |
| ## 🔐 Статус: СВОЙ (маяк активен) | |
| --- | |
| {SignatureGenerator.SIGNATURE} | |
| """ | |
| report_path = Path("INSTALL_REPORT.md") | |
| with open(report_path, 'w', encoding='utf-8') as f: | |
| f.write(report) | |
| SignatureGenerator.sign_file(report_path) | |
| return str(report_path) | |
| def run(self) -> bool: | |
| self.print_banner() | |
| try: | |
| CounterIntel.log_event('installer_started', {'version': self.VERSION}) | |
| if not self.get_user_input(): | |
| return False | |
| if not self.prepare_environment(): | |
| CounterIntel.log_event('installer_failed', {'stage': 'prepare'}) | |
| return False | |
| if not self.analyze_and_configure(): | |
| CounterIntel.log_event('installer_failed', {'stage': 'configure'}) | |
| return False | |
| if not self.deploy(): | |
| CounterIntel.log_event('installer_failed', {'stage': 'deploy'}) | |
| return False | |
| report_path = self.generate_report() | |
| print(f"\n📄 Отчёт сохранён: {report_path}") | |
| self.cleanup() | |
| CounterIntel.log_event('installer_completed', {'status': 'success'}) | |
| print("\n" + "="*70) | |
| print("✅ УСТАНОВКА ЗАВЕРШЕНА УСПЕШНО!") | |
| print("="*70) | |
| return True | |
| except Exception as e: | |
| print(f"\n❌ Критическая ошибка: {e}") | |
| CounterIntel.log_event('installer_error', {'error': str(e)}) | |
| self.cleanup() | |
| return False | |
| if __name__ == "__main__": | |
| installer = UniversalInstaller() | |
| success = installer.run() | |
| sys.exit(0 if success else 1) | |