rss-system / deploy_master.py
NIKOLAJ5313's picture
Добавил RSS System
16263d4
Raw History Blame Contribute Delete
25.2 kB
#!/usr/bin/env python3
# ============================================================================
# ╔═════════════════════════════════════════════════════════════════════════╗
# ║ ЦИФРОВАЯ ВСЕЛЕННАЯ — НИКОЛАЙ АЛЕКСАНДРОВИЧ ПОТЕХИН ║
# ║ (позывной: БАТЕНЬКА) ║
# ╠═════════════════════════════════════════════════════════════════════════╣
# ║ 🏛 Автор: Н.А. Потехин ║
# ║ 🔐 Ключ: 3f7c5d8e2a1b9f4a (hash of "бедра") ║
# ║ 🔎 Проверка: Контрразведка: base64.b64decode("0LXRhdGA0LA=") = бедра ║
# ║ 🕵️ Статус: СВОЙ (маяк активен) ║
# ╚═════════════════════════════════════════════════════════════════════════╝
# ============================================================================
import os
import sys
import json
import shutil
import subprocess
import hashlib
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional
# ============================================================================
# КОНТРРАЗВЕДКА: ПРОВЕРКА ЦЕЛОСТНОСТИ
# ============================================================================
class CounterIntel:
"""Контрразведка установщика"""
INSTALLER_HASH = "" # Заполняется при сборке
@staticmethod
def verify_file(filepath: str) -> bool:
"""Проверяет хэш файла"""
if not os.path.exists(filepath):
return False
with open(filepath, 'rb') as f:
return True # Упрощённая проверка
@staticmethod
def log_event(event_type: str, details: dict):
"""Логирует событие установки"""
timestamp = datetime.utcnow().isoformat()
log_entry = f"[{timestamp}] {event_type}: {json.dumps(details)}"
print(f"🕵️ {log_entry}")
@staticmethod
def check_suspicious_activity():
"""Проверка на подозрительную активность"""
suspicious_vars = ['MALWARE', 'INJECT', 'OVERRIDE']
for var in os.environ.keys():
if any(s in var.upper() for s in suspicious_vars):
CounterIntel.log_event('SUSPICIOUS_ENV', {'var': var})
return False
return True
# ============================================================================
# ПОГРАНИЧНЫЙ СЛОЙ: ВАЛИДАЦИЯ
# ============================================================================
class BorderGuard:
"""Пограничный контроль установщика"""
REQUIRED_TOKEN_PREFIXES = {
'github': 'ghp_',
'huggingface': 'hf_'
}
MIN_DISK_SPACE_MB = 1000
MIN_RAM_MB = 2048
@staticmethod
def validate_token(token: str, token_type: str) -> bool:
"""Проверяет валидность токена"""
if not token or len(token) < 20:
return False
prefix = BorderGuard.REQUIRED_TOKEN_PREFIXES.get(token_type, '')
if prefix and not token.startswith(prefix):
print(f"⚠️ Токен {token_type} не имеет ожидаемого префикса {prefix}")
return True
@staticmethod
def check_disk_space() -> bool:
"""Проверяет свободное место на диске"""
try:
total, used, free = shutil.disk_usage("/")
free_mb = free // (2**20)
if free_mb < BorderGuard.MIN_DISK_SPACE_MB:
print(f"❌ Недостаточно места: {free_mb}MB (нужно {BorderGuard.MIN_DISK_SPACE_MB}MB)")
return False
print(f"✅ Место на диске: {free_mb}MB")
return True
except Exception as e:
print(f"⚠️ Не удалось проверить диск: {e}")
return True
@staticmethod
def check_python_version() -> bool:
"""Проверяет версию Python"""
version = sys.version_info
if version.major < 3 or version.minor < 8:
print(f"❌ Python {version.major}.{version.minor} не поддерживается. Нужен 3.8+")
return False
print(f"✅ Python {version.major}.{version.minor}.{version.micro}")
return True
@staticmethod
def check_dependencies() -> List[str]:
"""Проверяет наличие необходимых утилит"""
required = ['git']
missing = []
for cmd in required:
try:
subprocess.run([cmd, '--version'], capture_output=True, check=True)
except (subprocess.CalledProcessError, FileNotFoundError):
missing.append(cmd)
if missing:
print(f"⚠️ Отсутствуют утилиты: {', '.join(missing)}")
else:
print(f"✅ Все зависимости найдены")
return missing
# ============================================================================
# ГЕНЕРАТОР ПОДПИСИ
# ============================================================================
class SignatureGenerator:
"""Генератор подписи Батеньки для всех файлов"""
SIGNATURE = """
# ============================================================================
# ╔═════════════════════════════════════════════════════════════════════════╗
# ║ ЦИФРОВАЯ ВСЕЛЕННАЯ — НИКОЛАЙ АЛЕКСАНДРОВИЧ ПОТЕХИН ║
# ║ (позывной: БАТЕНЬКА) ║
# ╠═════════════════════════════════════════════════════════════════════════╣
# ║ 🏛 Автор: Н.А. Потехин ║
# ║ 🔐 Ключ: 3f7c5d8e2a1b9f4a (hash of "бедра") ║
# ║ 🔎 Проверка: Контрразведка: base64.b64decode("0LXRhdGA0LA=") = бедра ║
# ║ 🕵️ Статус: СВОЙ (маяк активен) ║
# ╚═════════════════════════════════════════════════════════════════════════╝
# ============================================================================
"""
@staticmethod
def sign_file(filepath: str):
"""Добавляет подпись в конец файла"""
try:
with open(filepath, 'a', encoding='utf-8') as f:
f.write("\n")
f.write(SignatureGenerator.SIGNATURE)
except Exception as e:
print(f"⚠️ Не удалось подписать файл {filepath}: {e}")
# ============================================================================
# АНАЛИЗАТОР СЛОЖНОСТИ
# ============================================================================
class ComplexityAnalyzer:
"""Анализирует систему и определяет стратегию развёртывания"""
HF_RAM_LIMIT_MB = 16000
HF_CPU_LIMIT = 2
CONTAINER_OVERHEAD_MB = 1000
COMPLEXITY_THRESHOLDS = {
'files_count': 15,
'size_mb': 50,
'parsers_count': 3
}
def __init__(self, source_dir: Path):
self.source_dir = source_dir
def analyze(self) -> Dict:
files = list(self.source_dir.rglob("*.py"))
total_size = sum(f.stat().st_size for f in self.source_dir.rglob("*"))
size_mb = total_size / (1024 * 1024)
parsers_count = len(list(self.source_dir.rglob("parser*.py")))
result = {
'files_count': len(files),
'size_mb': round(size_mb, 2),
'parsers_count': parsers_count,
'mode': 'single_container',
'containers': 1,
'ram_per_container': self.HF_RAM_LIMIT_MB - self.CONTAINER_OVERHEAD_MB
}
if (len(files) > self.COMPLEXITY_THRESHOLDS['files_count'] or
size_mb > self.COMPLEXITY_THRESHOLDS['size_mb'] or
parsers_count > self.COMPLEXITY_THRESHOLDS['parsers_count']):
result['mode'] = 'multi_container'
result['containers'] = 3
result['ram_per_container'] = (self.HF_RAM_LIMIT_MB - self.CONTAINER_OVERHEAD_MB) // 3
print(f"📊 Анализ: {result['files_count']} файлов, {result['size_mb']}MB, режим: {result['mode']}")
return result
# ============================================================================
# ГЕНЕРАТОР КОНФИГУРАЦИИ
# ============================================================================
class ConfigGenerator:
def __init__(self, analysis: Dict, gh_token: str, hf_token: str, gh_repo: str, hf_repo: str):
self.analysis = analysis
self.gh_token = gh_token
self.hf_token = hf_token
self.gh_repo = gh_repo
self.hf_repo = hf_repo
def generate_dockerfile(self, output_path: Path):
content = """FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app
USER appuser
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\
CMD python -c "import os; exit(0 if os.path.exists('output/feed.xml') else 1)"
EXPOSE 7860
CMD ["python", "app.py"]
"""
with open(output_path / "Dockerfile", 'w', encoding='utf-8') as f:
f.write(content)
SignatureGenerator.sign_file(output_path / "Dockerfile")
print("✅ Dockerfile сгенерирован")
def generate_docker_compose(self, output_path: Path):
ram_limit = self.analysis['ram_per_container']
cpu_limit = 2.0 / self.analysis['containers']
if self.analysis['mode'] == 'multi_container':
services = """
worker:
build: .
environment:
- SERVICE_TYPE=worker
- MAX_CONCURRENT=3
deploy:
resources:
limits:
memory: ${RAM_LIMIT}M
cpus: '${CPU_LIMIT}'
restart: always
monitor:
build: .
environment:
- SERVICE_TYPE=monitor
- GRADIO_SERVER_NAME=0.0.0.0
- GRADIO_SERVER_PORT=7860
ports:
- "7860:7860"
deploy:
resources:
limits:
memory: ${RAM_LIMIT}M
cpus: '${CPU_LIMIT}'
restart: always
api:
build: .
environment:
- SERVICE_TYPE=api
- API_PORT=8000
deploy:
resources:
limits:
memory: ${RAM_LIMIT}M
cpus: '${CPU_LIMIT}'
restart: always
"""
else:
services = """
app:
build: .
environment:
- SERVICE_TYPE=single
- GRADIO_SERVER_NAME=0.0.0.0
- GRADIO_SERVER_PORT=7860
ports:
- "7860:7860"
deploy:
resources:
limits:
memory: ${RAM_LIMIT}M
cpus: '${CPU_LIMIT}'
restart: always
"""
content = f"""version: '3.8'
services:
{services}
networks:
rss-network:
driver: bridge
volumes:
output:
config:
logs:
x-env: &default-env
RAM_LIMIT: {ram_limit}
CPU_LIMIT: {cpu_limit}
"""
with open(output_path / "docker-compose.yml", 'w', encoding='utf-8') as f:
f.write(content)
SignatureGenerator.sign_file(output_path / "docker-compose.yml")
print("✅ docker-compose.yml сгенерирован")
def generate_github_workflow(self, output_path: Path):
content = """name: 🔄 RSS Generator & Deploy
on:
schedule:
- cron: '0 */6 * * *'
workflow_dispatch:
env:
PYTHON_VERSION: '3.11'
jobs:
generate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- run: pip install -r requirements.txt
- run: python -c "from core.self_heal import create_baseline; create_baseline()"
- run: python parsers/text_parser.py || true
- run: python parsers/video_parser.py || true
- run: python parsers/audio_parser.py || true
- run: python -c "from core.mixer import main; main()"
- run: python -c "from core.rss_generator import main; main()"
- run: python -c "from core.signature import sign_file; sign_file('output/feed.xml')"
- uses: peaceiris/actions-gh-pages@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./output
keep_files: true
"""
workflows_dir = output_path / ".github" / "workflows"
workflows_dir.mkdir(parents=True, exist_ok=True)
with open(workflows_dir / "rss.yml", 'w', encoding='utf-8') as f:
f.write(content)
SignatureGenerator.sign_file(workflows_dir / "rss.yml")
print("✅ GitHub Actions workflow сгенерирован")
def generate_env_template(self, output_path: Path):
content = f"""# GitHub Configuration
GH_TOKEN={self.gh_token[:10]}...
GH_REPO={self.gh_repo}
# Hugging Face Configuration
HF_TOKEN={self.hf_token[:10]}...
HF_REPO={self.hf_repo}
# API Keys
YOUTUBE_API_KEY=
PODCAST_INDEX_KEY=
# Alerts
ALERT_WEBHOOK=
ALERT_EMAIL=
# System
ENV=production
LOG_LEVEL=INFO
"""
with open(output_path / ".env.template", 'w', encoding='utf-8') as f:
f.write(content)
print("✅ .env.template сгенерирован")
# ============================================================================
# ДЕПЛОЕР
# ============================================================================
class Deployer:
def __init__(self, gh_token: str, hf_token: str, gh_repo: str, hf_repo: str, source_dir: Path):
self.gh_token = gh_token
self.hf_token = hf_token
self.gh_repo = gh_repo
self.hf_repo = hf_repo
self.source_dir = source_dir
def push_to_github(self) -> bool:
print("🐙 Деплой на GitHub...")
try:
os.chdir(self.source_dir)
subprocess.run(["git", "init"], check=True, capture_output=True)
subprocess.run(["git", "remote", "remove", "origin"], capture_output=True)
subprocess.run(
["git", "remote", "add", "origin",
f"https://{self.gh_token}@github.com/{self.gh_repo}.git"],
check=True, capture_output=True
)
subprocess.run(["git", "add", "."], check=True, capture_output=True)
subprocess.run(
["git", "commit", "-m", "🚀 Deploy by Batenka Installer"],
check=True, capture_output=True
)
subprocess.run(
["git", "push", "-u", "origin", "main", "--force"],
check=True, capture_output=True
)
print("✅ GitHub деплой успешен")
return True
except Exception as e:
print(f"❌ Ошибка GitHub деплоя: {e}")
return False
def push_to_huggingface(self) -> bool:
print("🤗 Деплой на Hugging Face Spaces...")
try:
os.chdir(self.source_dir)
subprocess.run(["git", "remote", "remove", "hf"], capture_output=True)
subprocess.run(
["git", "remote", "add", "hf",
f"https://user:{self.hf_token}@huggingface.co/spaces/{self.hf_repo}"],
check=True, capture_output=True
)
subprocess.run(
["git", "push", "hf", "main", "--force"],
check=True, capture_output=True
)
print("✅ Hugging Face деплой успешен")
return True
except Exception as e:
print(f"❌ Ошибка HF деплоя: {e}")
return False
# ============================================================================
# ГЛАВНЫЙ КЛАСС УСТАНОВЩИКА
# ============================================================================
class UniversalInstaller:
VERSION = "1.0.0"
def __init__(self):
self.config = {}
self.temp_dir = Path("./deploy_temp")
self.source_dir = Path("./src")
def print_banner(self):
banner = """
╔═══════════════════════════════════════════════════════════════════════════╗
║ УНИВЕРСАЛЬНЫЙ УСТАНОВЩИК RSS ║
║ (СИСТЕМА БАТЕНЬКИ) ║
║ Версия: 1.0.0 | Автор: Н.А. Потехин (БАТЕНЬКА) | Статус: СВОЙ ║
╚═══════════════════════════════════════════════════════════════════════════╝
"""
print(banner)
def get_user_input(self) -> bool:
print("\n🔐 Ввод учётных данных...\n")
try:
self.config['gh_token'] = input("GitHub Token (ghp_...): ").strip()
self.config['hf_token'] = input("Hugging Face Token (hf_...): ").strip()
self.config['gh_repo'] = input("GitHub Repo (user/repo-name): ").strip()
self.config['hf_repo'] = input("HF Space Repo (user/space-name): ").strip()
self.config['source_dir'] = input("Путь к исходникам [./src]: ").strip() or "./src"
if not all([self.config['gh_token'], self.config['hf_token'],
self.config['gh_repo'], self.config['hf_repo']]):
print("❌ Все поля обязательны")
return False
BorderGuard.validate_token(self.config['gh_token'], 'github')
BorderGuard.validate_token(self.config['hf_token'], 'huggingface')
return True
except KeyboardInterrupt:
print("\n❌ Установка прервана")
return False
except Exception as e:
print(f"❌ Ошибка ввода: {e}")
return False
def prepare_environment(self) -> bool:
print("\n🛠 Подготовка среды...\n")
if not BorderGuard.check_python_version():
return False
if not BorderGuard.check_disk_space():
return False
if not CounterIntel.check_suspicious_activity():
return False
if self.temp_dir.exists():
shutil.rmtree(self.temp_dir)
self.temp_dir.mkdir()
source_path = Path(self.config['source_dir'])
if not source_path.exists():
print(f"❌ Исходники не найдены: {source_path}")
return False
shutil.copytree(source_path, self.temp_dir / "src", dirs_exist_ok=True)
self.source_dir = self.temp_dir / "src"
print("✅ Среда подготовлена")
return True
def analyze_and_configure(self) -> bool:
print("\n📊 Анализ системы...\n")
analyzer = ComplexityAnalyzer(self.source_dir)
analysis = analyzer.analyze()
generator = ConfigGenerator(
analysis=analysis,
gh_token=self.config['gh_token'],
hf_token=self.config['hf_token'],
gh_repo=self.config['gh_repo'],
hf_repo=self.config['hf_repo']
)
generator.generate_dockerfile(self.source_dir)
generator.generate_docker_compose(self.source_dir)
generator.generate_github_workflow(self.source_dir)
generator.generate_env_template(self.source_dir)
self.analysis = analysis
print("✅ Конфигурация сгенерирована")
return True
def deploy(self) -> bool:
print("\n🚀 Деплой системы...\n")
deployer = Deployer(
self.config['gh_token'], self.config['hf_token'],
self.config['gh_repo'], self.config['hf_repo'],
self.source_dir
)
gh_success = deployer.push_to_github()
hf_success = deployer.push_to_huggingface()
if gh_success and hf_success:
print("✅ Деплой успешен")
return True
elif gh_success or hf_success:
print("⚠️ Деплой частичен")
return True
else:
print("❌ Деплой не удался")
return False
def cleanup(self):
print("\n🧹 Очистка...\n")
if self.temp_dir.exists():
shutil.rmtree(self.temp_dir)
print("✅ Временные файлы удалены")
def generate_report(self) -> str:
report = f"""
# 📋 ОТЧЁТ ОБ УСТАНОВКЕ RSS-СИСТЕМЫ
**Дата:** {datetime.utcnow().isoformat()}
**Установщик:** UniversalInstaller v{self.VERSION}
**Статус:** ✅ УСПЕШНО
## 🔧 Конфигурация:
- **Режим:** {self.analysis.get('mode', 'unknown')}
- **Контейнеров:** {self.analysis.get('containers', 1)}
- **RAM на контейнер:** {self.analysis.get('ram_per_container', 0)}MB
## 🔗 Ссылки:
- **GitHub Pages:** https://{self.config['gh_repo'].split('/')[0]}.github.io/{self.config['gh_repo'].split('/')[1]}/feed.xml
- **Hugging Face:** https://huggingface.co/spaces/{self.config['hf_repo']}
- **GitHub Actions:** https://github.com/{self.config['gh_repo']}/actions
## 🔐 Статус: СВОЙ (маяк активен)
---
{SignatureGenerator.SIGNATURE}
"""
report_path = Path("INSTALL_REPORT.md")
with open(report_path, 'w', encoding='utf-8') as f:
f.write(report)
SignatureGenerator.sign_file(report_path)
return str(report_path)
def run(self) -> bool:
self.print_banner()
try:
CounterIntel.log_event('installer_started', {'version': self.VERSION})
if not self.get_user_input():
return False
if not self.prepare_environment():
CounterIntel.log_event('installer_failed', {'stage': 'prepare'})
return False
if not self.analyze_and_configure():
CounterIntel.log_event('installer_failed', {'stage': 'configure'})
return False
if not self.deploy():
CounterIntel.log_event('installer_failed', {'stage': 'deploy'})
return False
report_path = self.generate_report()
print(f"\n📄 Отчёт сохранён: {report_path}")
self.cleanup()
CounterIntel.log_event('installer_completed', {'status': 'success'})
print("\n" + "="*70)
print("✅ УСТАНОВКА ЗАВЕРШЕНА УСПЕШНО!")
print("="*70)
return True
except Exception as e:
print(f"\n❌ Критическая ошибка: {e}")
CounterIntel.log_event('installer_error', {'error': str(e)})
self.cleanup()
return False
if __name__ == "__main__":
installer = UniversalInstaller()
success = installer.run()
sys.exit(0 if success else 1)