Spaces:
Sleeping
Sleeping
Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,564 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import asyncio
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
import tempfile
|
| 8 |
+
import threading
|
| 9 |
+
import uuid
|
| 10 |
+
from io import StringIO
|
| 11 |
+
|
| 12 |
+
# --- Flask and Jinja Imports ---
|
| 13 |
+
from flask import Flask, render_template, request, redirect, url_for, flash
|
| 14 |
+
from jinja2 import BaseLoader, TemplateNotFound
|
| 15 |
+
|
| 16 |
+
# --- Core Workflow Imports (Embedded) ---
|
| 17 |
+
from telethon import TelegramClient
|
| 18 |
+
from googleapiclient.discovery import build
|
| 19 |
+
from googleapiclient.errors import HttpError
|
| 20 |
+
from googleapiclient.http import MediaFileUpload
|
| 21 |
+
from google.auth.transport.requests import Request
|
| 22 |
+
from google.oauth2.credentials import Credentials
|
| 23 |
+
from google_auth_oauthlib.flow import InstalledAppFlow
|
| 24 |
+
import firebase_admin
|
| 25 |
+
from firebase_admin import credentials, db
|
| 26 |
+
|
| 27 |
+
# --- Custom Exception for Auth Flow ---
|
| 28 |
+
class GoogleAuthError(Exception):
|
| 29 |
+
pass
|
| 30 |
+
|
| 31 |
+
# --- Custom Jinja Loader for Embedded Templates ---
|
| 32 |
+
class StringLoader(BaseLoader):
|
| 33 |
+
"""A Jinja loader that loads templates from a dictionary."""
|
| 34 |
+
def __init__(self, templates):
|
| 35 |
+
self.templates = templates
|
| 36 |
+
|
| 37 |
+
def get_source(self, environment, template):
|
| 38 |
+
if template in self.templates:
|
| 39 |
+
source = self.templates[template]
|
| 40 |
+
return source, None, lambda: True
|
| 41 |
+
raise TemplateNotFound(template)
|
| 42 |
+
|
| 43 |
+
# --- Flask App Setup ---
|
| 44 |
+
app = Flask(__name__)
|
| 45 |
+
app.secret_key = 'your_strong_secret_key_change_this_for_production'
|
| 46 |
+
|
| 47 |
+
# --- In-Memory Storage for Logs & Status ---
|
| 48 |
+
global_logs = {}
|
| 49 |
+
processing_status = {}
|
| 50 |
+
|
| 51 |
+
# --- Custom Logging Handler for Web Capture ---
|
| 52 |
+
class InMemoryHandler(logging.Handler):
|
| 53 |
+
"""Custom logging handler to capture logs in memory for a specific run."""
|
| 54 |
+
def __init__(self, run_id):
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.run_id = run_id
|
| 57 |
+
|
| 58 |
+
def emit(self, record):
|
| 59 |
+
log_entry = self.format(record)
|
| 60 |
+
if self.run_id not in global_logs:
|
| 61 |
+
global_logs[self.run_id] = []
|
| 62 |
+
global_logs[self.run_id].append(log_entry)
|
| 63 |
+
|
| 64 |
+
# --- Embedded Core Workflow Code ---
|
| 65 |
+
workflow_logger = logging.getLogger('workflow_logger')
|
| 66 |
+
workflow_logger.setLevel(logging.INFO)
|
| 67 |
+
if not workflow_logger.handlers:
|
| 68 |
+
workflow_logger.addHandler(logging.StreamHandler())
|
| 69 |
+
|
| 70 |
+
class TelegramYouTubeWorkflow:
|
| 71 |
+
def __init__(self, config_dict, logger_instance=None):
|
| 72 |
+
self.config = config_dict
|
| 73 |
+
self.logger = logger_instance if logger_instance else workflow_logger
|
| 74 |
+
self.telegram_client = None
|
| 75 |
+
self.youtube_service = None
|
| 76 |
+
self.firebase_db = None
|
| 77 |
+
self.download_dir = Path(self.config.get('download_directory', 'downloads'))
|
| 78 |
+
self.download_dir.mkdir(exist_ok=True)
|
| 79 |
+
self.SCOPES = ['https://www.googleapis.com/auth/youtube.upload']
|
| 80 |
+
|
| 81 |
+
def setup_firebase(self, service_account_content, database_url):
|
| 82 |
+
try:
|
| 83 |
+
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as tmp:
|
| 84 |
+
tmp.write(service_account_content)
|
| 85 |
+
tmp_path = tmp.name
|
| 86 |
+
try:
|
| 87 |
+
if not firebase_admin._apps:
|
| 88 |
+
cred = credentials.Certificate(tmp_path)
|
| 89 |
+
firebase_admin.initialize_app(cred, {'databaseURL': database_url})
|
| 90 |
+
self.firebase_db = db
|
| 91 |
+
self.logger.info("Connected to Firebase successfully")
|
| 92 |
+
finally:
|
| 93 |
+
os.unlink(tmp_path)
|
| 94 |
+
except Exception as e:
|
| 95 |
+
self.logger.error(f"Failed to connect to Firebase: {e}")
|
| 96 |
+
raise
|
| 97 |
+
|
| 98 |
+
async def setup_telegram_client(self):
|
| 99 |
+
# ... (rest of the class methods are unchanged)
|
| 100 |
+
telegram_config = self.config['telegram']
|
| 101 |
+
self.telegram_client = TelegramClient(
|
| 102 |
+
'session',
|
| 103 |
+
telegram_config['api_id'],
|
| 104 |
+
telegram_config['api_hash']
|
| 105 |
+
)
|
| 106 |
+
await self.telegram_client.start(phone=telegram_config['phone_number'])
|
| 107 |
+
self.logger.info("Connected to Telegram successfully")
|
| 108 |
+
|
| 109 |
+
def setup_youtube_client(self, client_secrets_content, token_path, auth_code=None):
|
| 110 |
+
"""Initialize YouTube API client using console flow for headless environments."""
|
| 111 |
+
creds = None
|
| 112 |
+
if os.path.exists(token_path) and os.path.getsize(token_path) > 0:
|
| 113 |
+
creds = Credentials.from_authorized_user_file(token_path, self.SCOPES)
|
| 114 |
+
|
| 115 |
+
if not creds or not creds.valid:
|
| 116 |
+
if creds and creds.expired and creds.refresh_token:
|
| 117 |
+
self.logger.info("Refreshing expired YouTube token...")
|
| 118 |
+
creds.refresh(Request())
|
| 119 |
+
else:
|
| 120 |
+
self.logger.info("No valid YouTube token found. Attempting console auth flow.")
|
| 121 |
+
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as tmp:
|
| 122 |
+
tmp.write(client_secrets_content)
|
| 123 |
+
secrets_path = tmp.name
|
| 124 |
+
|
| 125 |
+
try:
|
| 126 |
+
# Use the out-of-band (oob) redirect URI for the copy-paste flow
|
| 127 |
+
flow = InstalledAppFlow.from_client_secrets_file(
|
| 128 |
+
secrets_path, self.SCOPES, redirect_uri='urn:ietf:wg:oauth:2.0:oob'
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
if not auth_code:
|
| 132 |
+
# If no code is provided, generate URL and stop
|
| 133 |
+
auth_url, _ = flow.authorization_url(prompt='consent')
|
| 134 |
+
self.logger.error("="*80)
|
| 135 |
+
self.logger.error("USER ACTION REQUIRED FOR AUTHENTICATION:")
|
| 136 |
+
self.logger.error(f"\n1. Open this URL in your browser:\n\n{auth_url}\n")
|
| 137 |
+
self.logger.error("2. Authenticate and grant permissions.")
|
| 138 |
+
self.logger.error("3. Copy the authorization code Google provides.")
|
| 139 |
+
self.logger.error("4. Paste the code into the 'Google Auth Code' field on the main page and resubmit.")
|
| 140 |
+
self.logger.error("="*80)
|
| 141 |
+
raise GoogleAuthError("Authorization code required.")
|
| 142 |
+
else:
|
| 143 |
+
# If code is provided, fetch the token
|
| 144 |
+
self.logger.info("Authorization code provided. Fetching token...")
|
| 145 |
+
flow.fetch_token(code=auth_code)
|
| 146 |
+
creds = flow.credentials
|
| 147 |
+
finally:
|
| 148 |
+
os.unlink(secrets_path)
|
| 149 |
+
|
| 150 |
+
# Save the new credentials
|
| 151 |
+
with open(token_path, 'w') as token:
|
| 152 |
+
token.write(creds.to_json())
|
| 153 |
+
self.logger.info("YouTube token saved successfully.")
|
| 154 |
+
|
| 155 |
+
self.youtube_service = build('youtube', 'v3', credentials=creds)
|
| 156 |
+
self.logger.info("YouTube API client initialized successfully")
|
| 157 |
+
|
| 158 |
+
def is_video_processed(self, telegram_video_id, channel_username):
|
| 159 |
+
try:
|
| 160 |
+
ref = self.firebase_db.reference(f'processed_videos/{channel_username}/{telegram_video_id}')
|
| 161 |
+
return ref.get() is not None
|
| 162 |
+
except Exception as e:
|
| 163 |
+
self.logger.error(f"Error checking processed video: {e}")
|
| 164 |
+
return False
|
| 165 |
+
|
| 166 |
+
def save_processed_video(self, telegram_video_id, channel_username, youtube_video_id=None, video_info=None):
|
| 167 |
+
try:
|
| 168 |
+
telegram_url = f"https://t.me/{channel_username.lstrip('@')}/{telegram_video_id}"
|
| 169 |
+
document = {
|
| 170 |
+
"telegram_video_id": telegram_video_id,
|
| 171 |
+
"channel_username": channel_username,
|
| 172 |
+
"telegram_url": telegram_url,
|
| 173 |
+
"youtube_video_id": youtube_video_id,
|
| 174 |
+
"processed_at": datetime.utcnow().isoformat(),
|
| 175 |
+
"status": "uploaded" if youtube_video_id else "failed"
|
| 176 |
+
}
|
| 177 |
+
if video_info:
|
| 178 |
+
document.update({
|
| 179 |
+
"video_title": video_info.get('caption', '')[:100],
|
| 180 |
+
"video_date": video_info.get('date').isoformat(),
|
| 181 |
+
"video_size": getattr(video_info.get('video'), 'size', None),
|
| 182 |
+
"video_duration": getattr(video_info.get('video'), 'duration', None)
|
| 183 |
+
})
|
| 184 |
+
ref = self.firebase_db.reference(f'processed_videos/{channel_username}/{telegram_video_id}')
|
| 185 |
+
ref.set(document)
|
| 186 |
+
self.logger.info(f"Saved processed video info to Firebase: {telegram_url}")
|
| 187 |
+
except Exception as e:
|
| 188 |
+
self.logger.error(f"Error saving processed video info: {e}")
|
| 189 |
+
|
| 190 |
+
def get_processed_videos_stats(self):
|
| 191 |
+
try:
|
| 192 |
+
ref = self.firebase_db.reference('processed_videos')
|
| 193 |
+
all_videos = ref.get()
|
| 194 |
+
if not all_videos: return {"total": 0, "uploaded": 0, "failed": 0}
|
| 195 |
+
total = 0
|
| 196 |
+
uploaded = 0
|
| 197 |
+
for channel in all_videos.values():
|
| 198 |
+
for video in channel.values():
|
| 199 |
+
total += 1
|
| 200 |
+
if video.get("status") == "uploaded":
|
| 201 |
+
uploaded += 1
|
| 202 |
+
failed = total - uploaded
|
| 203 |
+
self.logger.info(f"DB stats - Total: {total}, Uploaded: {uploaded}, Failed: {failed}")
|
| 204 |
+
return {"total": total, "uploaded": uploaded, "failed": failed}
|
| 205 |
+
except Exception as e:
|
| 206 |
+
self.logger.error(f"Error getting stats: {e}")
|
| 207 |
+
return None
|
| 208 |
+
|
| 209 |
+
async def get_channel_videos(self, limit=10):
|
| 210 |
+
channel_username = self.config['telegram']['channel_username']
|
| 211 |
+
try:
|
| 212 |
+
entity = await self.telegram_client.get_entity(channel_username)
|
| 213 |
+
videos = []
|
| 214 |
+
async for message in self.telegram_client.iter_messages(entity, limit=limit):
|
| 215 |
+
if message.video and not self.is_video_processed(message.id, channel_username):
|
| 216 |
+
videos.append({
|
| 217 |
+
'id': message.id, 'message': message, 'video': message.video,
|
| 218 |
+
'caption': message.text or '', 'date': message.date,
|
| 219 |
+
'telegram_url': f"https://t.me/{channel_username.lstrip('@')}/{message.id}"
|
| 220 |
+
})
|
| 221 |
+
self.logger.info(f"Found {len(videos)} new videos to process.")
|
| 222 |
+
return videos
|
| 223 |
+
except Exception as e:
|
| 224 |
+
self.logger.error(f"Error getting channel videos: {e}")
|
| 225 |
+
return []
|
| 226 |
+
|
| 227 |
+
async def download_video(self, video_info):
|
| 228 |
+
try:
|
| 229 |
+
filename = f"video_{video_info['id']}.mp4"
|
| 230 |
+
filepath = self.download_dir / filename
|
| 231 |
+
self.logger.info(f"Downloading video {video_info['id']}...")
|
| 232 |
+
await video_info['message'].download_media(file=str(filepath))
|
| 233 |
+
self.logger.info(f"Downloaded to: {filepath}")
|
| 234 |
+
return str(filepath)
|
| 235 |
+
except Exception as e:
|
| 236 |
+
self.logger.error(f"Error downloading video {video_info['id']}: {e}")
|
| 237 |
+
return None
|
| 238 |
+
|
| 239 |
+
def upload_to_youtube(self, video_path, video_info):
|
| 240 |
+
try:
|
| 241 |
+
video_settings = self.config['video_settings']
|
| 242 |
+
title = f"{video_settings.get('title_prefix', '')}{video_info['caption'][:100]}" or f"Video from Telegram {video_info['date'].strftime('%Y-%m-%d')}"
|
| 243 |
+
body = {
|
| 244 |
+
'snippet': {
|
| 245 |
+
'title': title,
|
| 246 |
+
'description': f"{video_settings.get('description_template', '')}\n\n{video_info['caption']}",
|
| 247 |
+
'tags': video_settings.get('tags', []),
|
| 248 |
+
'categoryId': video_settings.get('category_id', '22')
|
| 249 |
+
},
|
| 250 |
+
'status': {'privacyStatus': video_settings.get('privacy_status', 'private')}
|
| 251 |
+
}
|
| 252 |
+
media = MediaFileUpload(video_path, chunksize=-1, resumable=True)
|
| 253 |
+
request = self.youtube_service.videos().insert(part=','.join(body.keys()), body=body, media_body=media)
|
| 254 |
+
|
| 255 |
+
self.logger.info(f"Uploading {os.path.basename(video_path)} to YouTube...")
|
| 256 |
+
response = None
|
| 257 |
+
while response is None:
|
| 258 |
+
status, response = request.next_chunk()
|
| 259 |
+
if status:
|
| 260 |
+
self.logger.info(f"Upload progress: {int(status.progress() * 100)}%")
|
| 261 |
+
|
| 262 |
+
self.logger.info(f"Video uploaded successfully! YouTube ID: {response['id']}")
|
| 263 |
+
return response['id']
|
| 264 |
+
except HttpError as e:
|
| 265 |
+
self.logger.error(f"HTTP error during upload: {e.content}")
|
| 266 |
+
return None
|
| 267 |
+
except Exception as e:
|
| 268 |
+
self.logger.error(f"Error uploading to YouTube: {e}")
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
def cleanup_video(self, video_path):
|
| 272 |
+
try:
|
| 273 |
+
os.remove(video_path)
|
| 274 |
+
self.logger.info(f"Cleaned up: {video_path}")
|
| 275 |
+
except Exception as e:
|
| 276 |
+
self.logger.error(f"Error cleaning up {video_path}: {e}")
|
| 277 |
+
|
| 278 |
+
async def process_videos(self, limit=5):
|
| 279 |
+
self.logger.info("Starting video processing workflow...")
|
| 280 |
+
try:
|
| 281 |
+
await self.setup_telegram_client()
|
| 282 |
+
self.get_processed_videos_stats()
|
| 283 |
+
videos = await self.get_channel_videos(limit)
|
| 284 |
+
if not videos:
|
| 285 |
+
self.logger.info("No new videos found.")
|
| 286 |
+
return
|
| 287 |
+
|
| 288 |
+
for video_info in videos:
|
| 289 |
+
video_path = None
|
| 290 |
+
youtube_id = None
|
| 291 |
+
try:
|
| 292 |
+
self.logger.info(f"Processing video: {video_info['telegram_url']}")
|
| 293 |
+
video_path = await self.download_video(video_info)
|
| 294 |
+
if video_path:
|
| 295 |
+
youtube_id = self.upload_to_youtube(video_path, video_info)
|
| 296 |
+
except Exception as e:
|
| 297 |
+
self.logger.error(f"Unhandled error processing video {video_info['id']}: {e}")
|
| 298 |
+
finally:
|
| 299 |
+
self.save_processed_video(
|
| 300 |
+
video_info['id'], self.config['telegram']['channel_username'],
|
| 301 |
+
youtube_video_id=youtube_id, video_info=video_info
|
| 302 |
+
)
|
| 303 |
+
if video_path:
|
| 304 |
+
self.cleanup_video(video_path)
|
| 305 |
+
except Exception as e:
|
| 306 |
+
self.logger.error(f"Core workflow error: {e}", exc_info=True)
|
| 307 |
+
finally:
|
| 308 |
+
if self.telegram_client and self.telegram_client.is_connected():
|
| 309 |
+
await self.telegram_client.disconnect()
|
| 310 |
+
self.logger.info("Workflow finished.")
|
| 311 |
+
|
| 312 |
+
# --- Flask Threading Function ---
|
| 313 |
+
|
| 314 |
+
def run_workflow_in_thread(run_id, config_data, client_secrets_content, service_account_content, num_videos, token_path, auth_code):
|
| 315 |
+
log_handler = InMemoryHandler(run_id)
|
| 316 |
+
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
| 317 |
+
log_handler.setFormatter(formatter)
|
| 318 |
+
workflow_logger.addHandler(log_handler)
|
| 319 |
+
|
| 320 |
+
try:
|
| 321 |
+
processing_status[run_id] = "Running..."
|
| 322 |
+
with tempfile.TemporaryDirectory() as temp_dir:
|
| 323 |
+
config_data['download_directory'] = os.path.join(temp_dir, 'downloads')
|
| 324 |
+
os.makedirs(config_data['download_directory'], exist_ok=True)
|
| 325 |
+
|
| 326 |
+
original_cwd = os.getcwd()
|
| 327 |
+
try:
|
| 328 |
+
os.chdir(temp_dir)
|
| 329 |
+
workflow = TelegramYouTubeWorkflow(config_dict=config_data, logger_instance=workflow_logger)
|
| 330 |
+
workflow.setup_firebase(service_account_content, config_data['firebase']['database_url'])
|
| 331 |
+
workflow.setup_youtube_client(client_secrets_content, token_path, auth_code)
|
| 332 |
+
asyncio.run(workflow.process_videos(limit=num_videos))
|
| 333 |
+
processing_status[run_id] = "Completed."
|
| 334 |
+
except GoogleAuthError as e:
|
| 335 |
+
processing_status[run_id] = f"Authentication Required: {e}"
|
| 336 |
+
workflow_logger.error(f"Workflow stopped for user authentication.")
|
| 337 |
+
except Exception as e:
|
| 338 |
+
processing_status[run_id] = f"Failed: {str(e)}"
|
| 339 |
+
workflow_logger.error(f"Workflow thread failed: {e}", exc_info=True)
|
| 340 |
+
finally:
|
| 341 |
+
os.chdir(original_cwd)
|
| 342 |
+
except Exception as e:
|
| 343 |
+
processing_status[run_id] = f"Setup Failed: {str(e)}"
|
| 344 |
+
workflow_logger.error(f"Thread setup failed: {e}", exc_info=True)
|
| 345 |
+
finally:
|
| 346 |
+
workflow_logger.removeHandler(log_handler)
|
| 347 |
+
log_handler.close()
|
| 348 |
+
|
| 349 |
+
# --- Flask Routes ---
|
| 350 |
+
|
| 351 |
+
@app.route('/', methods=['GET', 'POST'])
|
| 352 |
+
def index():
|
| 353 |
+
if request.method == 'POST':
|
| 354 |
+
try:
|
| 355 |
+
# --- Firebase ---
|
| 356 |
+
firebase_database_url = request.form['firebase_database_url']
|
| 357 |
+
firebase_service_account_file = request.files['firebase_service_account']
|
| 358 |
+
if not firebase_service_account_file or firebase_service_account_file.filename == '':
|
| 359 |
+
flash('Firebase service account file is required.', 'error')
|
| 360 |
+
return redirect(request.url)
|
| 361 |
+
service_account_content = firebase_service_account_file.read().decode('utf-8')
|
| 362 |
+
|
| 363 |
+
# --- YouTube ---
|
| 364 |
+
client_secrets_file = request.files['youtube_client_secrets']
|
| 365 |
+
if not client_secrets_file or client_secrets_file.filename == '':
|
| 366 |
+
flash('YouTube client secrets file is required.', 'error')
|
| 367 |
+
return redirect(request.url)
|
| 368 |
+
client_secrets_content = client_secrets_file.read().decode('utf-8')
|
| 369 |
+
|
| 370 |
+
# --- Google Auth Code (for first run) ---
|
| 371 |
+
google_auth_code = request.form.get('google_auth_code', '').strip()
|
| 372 |
+
|
| 373 |
+
config_data = {
|
| 374 |
+
"telegram": {
|
| 375 |
+
"api_id": int(request.form['telegram_api_id']),
|
| 376 |
+
"api_hash": request.form['telegram_api_hash'],
|
| 377 |
+
"phone_number": request.form['telegram_phone_number'],
|
| 378 |
+
"channel_username": request.form['telegram_channel_username']
|
| 379 |
+
},
|
| 380 |
+
"firebase": { "database_url": firebase_database_url },
|
| 381 |
+
"video_settings": {
|
| 382 |
+
"title_prefix": request.form.get('title_prefix', ''),
|
| 383 |
+
"description_template": request.form.get('description_template', 'Video from Telegram'),
|
| 384 |
+
"tags": [tag.strip() for tag in request.form.get('tags', 'telegram,video').split(',') if tag.strip()],
|
| 385 |
+
"category_id": request.form.get('category_id', '22'),
|
| 386 |
+
"privacy_status": request.form.get('privacy_status', 'private')
|
| 387 |
+
}
|
| 388 |
+
}
|
| 389 |
+
num_videos = int(request.form.get('num_videos', 5))
|
| 390 |
+
run_id = str(uuid.uuid4())
|
| 391 |
+
global_logs[run_id] = []
|
| 392 |
+
processing_status[run_id] = "Starting..."
|
| 393 |
+
|
| 394 |
+
token_fd, token_path = tempfile.mkstemp(suffix='.json', prefix='token_')
|
| 395 |
+
os.close(token_fd)
|
| 396 |
+
|
| 397 |
+
thread = threading.Thread(
|
| 398 |
+
target=run_workflow_in_thread,
|
| 399 |
+
args=(run_id, config_data, client_secrets_content, service_account_content, num_videos, token_path, google_auth_code)
|
| 400 |
+
)
|
| 401 |
+
thread.start()
|
| 402 |
+
|
| 403 |
+
flash(f'Workflow started with ID: {run_id}', 'info')
|
| 404 |
+
return redirect(url_for('results', run_id=run_id))
|
| 405 |
+
|
| 406 |
+
except ValueError as e:
|
| 407 |
+
flash(f'Invalid input data: {e}', 'error')
|
| 408 |
+
return redirect(request.url)
|
| 409 |
+
except Exception as e:
|
| 410 |
+
flash(f'Error starting workflow: {e}', 'error')
|
| 411 |
+
app.logger.error(f"Form processing error: {e}", exc_info=True)
|
| 412 |
+
return redirect(request.url)
|
| 413 |
+
|
| 414 |
+
return render_template('index.html')
|
| 415 |
+
|
| 416 |
+
@app.route('/results/<run_id>')
|
| 417 |
+
def results(run_id):
|
| 418 |
+
logs = global_logs.get(run_id, [])
|
| 419 |
+
status = processing_status.get(run_id, "Not Found")
|
| 420 |
+
return render_template('results.html', run_id=run_id, logs=logs, status=status)
|
| 421 |
+
|
| 422 |
+
# --- HTML Templates as Strings ---
|
| 423 |
+
|
| 424 |
+
index_template_content = '''<!doctype html>
|
| 425 |
+
<html lang="en">
|
| 426 |
+
<head>
|
| 427 |
+
<meta charset="utf-8">
|
| 428 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 429 |
+
<title>Telegram to YouTube Uploader</title>
|
| 430 |
+
<style>
|
| 431 |
+
|
| 432 |
+
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; margin: 2em; background-color: #f8f9fa; color: #212529; }
|
| 433 |
+
.container { max-width: 800px; margin: auto; background: #fff; padding: 2em; border-radius: 0.5em; box-shadow: 0 0 1em rgba(0,0,0,0.1); }
|
| 434 |
+
h1, h2 { color: #007bff; }
|
| 435 |
+
h2.auth-header { color: #dc3545; }
|
| 436 |
+
.form-group { margin-bottom: 1.5em; }
|
| 437 |
+
label { display: block; margin-bottom: 0.5em; font-weight: bold; }
|
| 438 |
+
input[type="text"], input[type="number"], input[type="password"], textarea, select {
|
| 439 |
+
width: 100%; padding: 0.75em; box-sizing: border-box; border: 1px solid #ced4da; border-radius: 0.25em;
|
| 440 |
+
}
|
| 441 |
+
button { background-color: #007bff; color: white; padding: 0.75em 1.5em; border: none; cursor: pointer; border-radius: 0.25em; font-size: 1em; }
|
| 442 |
+
button:hover { background-color: #0056b3; }
|
| 443 |
+
.flash-messages { margin-bottom: 1.5em; }
|
| 444 |
+
.flash-message { padding: 1em; margin-bottom: 1em; border-radius: 0.25em; }
|
| 445 |
+
.flash-message-info { background-color: #d1ecf1; border: 1px solid #bee5eb; color: #0c5460; }
|
| 446 |
+
.flash-message-error { background-color: #f8d7da; border: 1px solid #f5c6cb; color: #721c24; }
|
| 447 |
+
small { color: #6c757d; }
|
| 448 |
+
.auth-box { border: 2px solid #fd7e14; padding: 1em; border-radius: 0.5em; margin-bottom: 1.5em; background-color: #fff3cd; }
|
| 449 |
+
</style>
|
| 450 |
+
</head>
|
| 451 |
+
<body>
|
| 452 |
+
<div class="container">
|
| 453 |
+
<h1>Telegram to YouTube Uploader</h1>
|
| 454 |
+
<div class="flash-messages">
|
| 455 |
+
{% with messages = get_flashed_messages(with_categories=true) %}
|
| 456 |
+
{% if messages %}
|
| 457 |
+
{% for category, message in messages %}
|
| 458 |
+
<div class="flash-message flash-message-{{ category }}">{{ message }}</div>
|
| 459 |
+
{% endfor %}
|
| 460 |
+
{% endif %}
|
| 461 |
+
{% endwith %}
|
| 462 |
+
</div>
|
| 463 |
+
<form method="post" enctype="multipart/form-data">
|
| 464 |
+
<div class="auth-box">
|
| 465 |
+
<h2 class="auth-header">Google Authentication</h2>
|
| 466 |
+
<div class="form-group">
|
| 467 |
+
<label for="google_auth_code">Google Auth Code (only needed for first time setup)</label>
|
| 468 |
+
<input type="text" id="google_auth_code" name="google_auth_code" placeholder="Paste code here if requested in logs">
|
| 469 |
+
<small>If the logs ask for an auth code, paste it here and resubmit the form.</small>
|
| 470 |
+
</div>
|
| 471 |
+
</div>
|
| 472 |
+
|
| 473 |
+
<h2>Cloud Settings</h2>
|
| 474 |
+
<div class="form-group">
|
| 475 |
+
<label for="firebase_database_url">Firebase Realtime Database URL:</label>
|
| 476 |
+
<input type="text" id="firebase_database_url" name="firebase_database_url" required placeholder="https://your-project-id-default-rtdb.firebaseio.com">
|
| 477 |
+
</div>
|
| 478 |
+
<div class="form-group">
|
| 479 |
+
<label for="firebase_service_account">Firebase Service Account File (JSON):</label>
|
| 480 |
+
<input type="file" id="firebase_service_account" name="firebase_service_account" accept=".json" required>
|
| 481 |
+
</div>
|
| 482 |
+
<div class="form-group">
|
| 483 |
+
<label for="youtube_client_secrets">YouTube Client Secrets File (JSON):</label>
|
| 484 |
+
<input type="file" id="youtube_client_secrets" name="youtube_client_secrets" accept=".json" required>
|
| 485 |
+
</div>
|
| 486 |
+
|
| 487 |
+
<h2>Telegram Settings</h2>
|
| 488 |
+
<div class="form-group"><label for="telegram_api_id">API ID:</label><input type="number" id="telegram_api_id" name="telegram_api_id" required></div>
|
| 489 |
+
<div class="form-group"><label for="telegram_api_hash">API Hash:</label><input type="text" id="telegram_api_hash" name="telegram_api_hash" required></div>
|
| 490 |
+
<div class="form-group"><label for="telegram_phone_number">Phone Number:</label><input type="text" id="telegram_phone_number" name="telegram_phone_number" placeholder="+1234567890" required></div>
|
| 491 |
+
<div class="form-group"><label for="telegram_channel_username">Channel Username:</label><input type="text" id="telegram_channel_username" name="telegram_channel_username" placeholder="@channelname" required></div>
|
| 492 |
+
|
| 493 |
+
<h2>Processing Settings</h2>
|
| 494 |
+
<div class="form-group"><label for="num_videos">Videos to Check:</label><input type="number" id="num_videos" name="num_videos" value="5" min="1"></div>
|
| 495 |
+
<div class="form-group"><label for="title_prefix">YouTube Title Prefix:</label><input type="text" id="title_prefix" name="title_prefix"></div>
|
| 496 |
+
<div class="form-group"><label for="description_template">Description Template:</label><textarea id="description_template" name="description_template">Video from Telegram.</textarea></div>
|
| 497 |
+
<div class="form-group"><label for="tags">Tags (comma-separated):</label><input type="text" id="tags" name="tags" value="telegram,video,archive"></div>
|
| 498 |
+
<div class="form-group">
|
| 499 |
+
<label for="category_id">Category ID:</label><input type="text" id="category_id" name="category_id" value="22">
|
| 500 |
+
</div>
|
| 501 |
+
<div class="form-group">
|
| 502 |
+
<label for="privacy_status">Privacy Status:</label>
|
| 503 |
+
<select id="privacy_status" name="privacy_status"><option value="private" selected>Private</option><option value="public">Public</option><option value="unlisted">Unlisted</option></select>
|
| 504 |
+
</div>
|
| 505 |
+
<button type="submit">Start Workflow</button>
|
| 506 |
+
</form>
|
| 507 |
+
</div>
|
| 508 |
+
</body>
|
| 509 |
+
</html>
|
| 510 |
+
'''
|
| 511 |
+
|
| 512 |
+
results_template_content = '''<!doctype html>
|
| 513 |
+
<html lang="en">
|
| 514 |
+
<head>
|
| 515 |
+
<meta charset="utf-8">
|
| 516 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 517 |
+
<title>Workflow Results - {{ run_id }}</title>
|
| 518 |
+
<style>
|
| 519 |
+
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; margin: 2em; background-color: #f8f9fa; color: #212529; }
|
| 520 |
+
.container { max-width: 900px; margin: auto; background: #fff; padding: 2em; border-radius: 0.5em; box-shadow: 0 0 1em rgba(0,0,0,0.1); }
|
| 521 |
+
h1, h2 { color: #007bff; }
|
| 522 |
+
pre { background-color: #e9ecef; padding: 1em; border: 1px solid #ced4da; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word; border-radius: 0.25em; }
|
| 523 |
+
.status { margin-bottom: 1.5em; padding: 1em; border-radius: 0.25em; font-weight: bold; }
|
| 524 |
+
.status-Running { background-color: #d1ecf1; border: 1px solid #bee5eb; color: #0c5460; }
|
| 525 |
+
.status-Completed { background-color: #d4edda; border: 1px solid #c3e6cb; color: #155724; }
|
| 526 |
+
.status-Failed, .status-Setup, .status-Authentication { background-color: #f8d7da; border: 1px solid #f5c6cb; color: #721c24; }
|
| 527 |
+
.back-link { margin-top: 2em; display: inline-block; }
|
| 528 |
+
</style>
|
| 529 |
+
</head>
|
| 530 |
+
<body>
|
| 531 |
+
<div class="container">
|
| 532 |
+
<h1>Workflow Results</h1>
|
| 533 |
+
<p><strong>Run ID:</strong> {{ run_id }}</p>
|
| 534 |
+
<div class="status status-{{ status.split(':')[0] }}">
|
| 535 |
+
<strong>Status:</strong> {{ status }}
|
| 536 |
+
</div>
|
| 537 |
+
<div class="refresh-info">
|
| 538 |
+
<p>Page will auto-refresh every 5 seconds if the job is running.</p>
|
| 539 |
+
</div>
|
| 540 |
+
<h2>Logs:</h2>
|
| 541 |
+
{% if logs %}
|
| 542 |
+
<pre>{{ logs | join('\\n') }}</pre>
|
| 543 |
+
{% else %}
|
| 544 |
+
<p>No logs available yet. The process might be starting up.</p>
|
| 545 |
+
{% endif %}
|
| 546 |
+
<a href="{{ url_for('index') }}" class="back-link">Back to Start</a>
|
| 547 |
+
</div>
|
| 548 |
+
<script>
|
| 549 |
+
(function() {
|
| 550 |
+
const status = "{{ status }}";
|
| 551 |
+
if (status.includes("Running") || status.includes("Starting")) {
|
| 552 |
+
setTimeout(function() { location.reload(); }, 5000);
|
| 553 |
+
}
|
| 554 |
+
})();
|
| 555 |
+
</script>
|
| 556 |
+
</body>
|
| 557 |
+
</html>
|
| 558 |
+
'''
|
| 559 |
+
|
| 560 |
+
# --- Use the custom StringLoader for our embedded templates ---
|
| 561 |
+
app.jinja_loader = StringLoader({
|
| 562 |
+
'index.html': index_template_content,
|
| 563 |
+
'results.html': results_template_content,
|
| 564 |
+
})
|