Upload 170 files
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +3 -0
- assets/edu_note.wav +3 -0
- assets/fun_fact.wav +3 -0
- assets/thanks.wav +3 -0
- auth/database.py +88 -0
- auth/models.py +96 -0
- auth/routes.py +340 -0
- auth/schemas.py +59 -0
- auth/security.py +137 -0
- core/builders/api_builder.py +142 -0
- core/builders/docs_builder.py +89 -0
- core/builders/ui_builder.py +107 -0
- core/execution/context.py +155 -0
- core/execution/executor.py +166 -0
- core/registry/loader.py +45 -0
- core/registry/task_model.py +20 -0
- core/registry/tasks_registry.py +198 -0
- fonts/TikTok-Bold.ttf +3 -0
- ingestion/__init__.py +1 -0
- ingestion/base64_loader.py +19 -0
- ingestion/classifiers.py +29 -0
- ingestion/cookies/youtube.txt +210 -0
- ingestion/downloader.py +18 -0
- ingestion/normalizer.py +24 -0
- ingestion/resolver.py +127 -0
- ingestion/social.py +21 -0
- models/__init__.py +2 -0
- models/kokoro.py +125 -0
- models/tokenizer.py +238 -0
- publisher/__init__.py +1 -0
- publisher/account_manager.py +38 -0
- publisher/ai/gemini_client.py +83 -0
- publisher/bulk.py +187 -0
- publisher/hashtags.py +19 -0
- publisher/metadata_engine.py +24 -0
- publisher/models.py +17 -0
- publisher/oauth/providers/google.py +52 -0
- publisher/oauth/providers/meta.py +42 -0
- publisher/oauth/providers/tiktok.py +45 -0
- publisher/oauth/providers/youtube.py +8 -0
- publisher/oauth/router.py +47 -0
- publisher/oauth/sessions.py +29 -0
- publisher/oauth/storage.py +44 -0
- publisher/platform_dispatcher.py +48 -0
- publisher/platforms/__init__.py +1 -0
- publisher/platforms/auth.py +38 -0
- publisher/platforms/base.py +15 -0
- publisher/platforms/errors.py +14 -0
- publisher/platforms/facebook.py +28 -0
- publisher/platforms/http.py +38 -0
.gitattributes
CHANGED
|
@@ -35,3 +35,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
fonts/TikTok-Bold.ttf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
gradio_demo.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
fonts/TikTok-Bold.ttf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
gradio_demo.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
assets/edu_note.wav filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
assets/fun_fact.wav filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
assets/thanks.wav filter=lfs diff=lfs merge=lfs -text
|
assets/edu_note.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:880362273fe0348d8e73e9cf99cbe58573ccaac1e51628e92d4a362c8199d399
|
| 3 |
+
size 416444
|
assets/fun_fact.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:245b8db3fc90671b02baa477ce0c2d08754b225ae95a853687191ad126de562e
|
| 3 |
+
size 496844
|
assets/thanks.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d04843d86bff241ff659cf6c6931641797411dbf3b1532eee83914058bbc5a64
|
| 3 |
+
size 288044
|
auth/database.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from sqlalchemy import create_engine
|
| 3 |
+
from sqlalchemy.orm import sessionmaker, declarative_base
|
| 4 |
+
from sqlalchemy.exc import OperationalError
|
| 5 |
+
|
| 6 |
+
# ==========================================================
|
| 7 |
+
# DATABASE URL
|
| 8 |
+
# ==========================================================
|
| 9 |
+
DATABASE_URL = os.getenv("DATABASE_URL")
|
| 10 |
+
|
| 11 |
+
if not DATABASE_URL:
|
| 12 |
+
raise RuntimeError(
|
| 13 |
+
"DATABASE_URL environment variable is not set"
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
# ==========================================================
|
| 17 |
+
# SUPABASE ENTERPRISE ENGINE CONFIG
|
| 18 |
+
# ==========================================================
|
| 19 |
+
# Optimized for:
|
| 20 |
+
# - Supabase Pooler
|
| 21 |
+
# - FastAPI async workload
|
| 22 |
+
# - Background workers
|
| 23 |
+
# - Long-running autonomous services
|
| 24 |
+
|
| 25 |
+
engine = create_engine(
|
| 26 |
+
DATABASE_URL,
|
| 27 |
+
|
| 28 |
+
# --- Pool Stability ---
|
| 29 |
+
pool_pre_ping=True, # validates dead connections
|
| 30 |
+
pool_recycle=300, # refresh connections
|
| 31 |
+
pool_size=5, # safe baseline
|
| 32 |
+
max_overflow=10, # burst capacity
|
| 33 |
+
|
| 34 |
+
# --- Reliability ---
|
| 35 |
+
echo=False,
|
| 36 |
+
future=True,
|
| 37 |
+
|
| 38 |
+
# --- Supabase Requirement ---
|
| 39 |
+
connect_args={
|
| 40 |
+
"sslmode": "require",
|
| 41 |
+
"connect_timeout": 10,
|
| 42 |
+
},
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
# ==========================================================
|
| 46 |
+
# SESSION FACTORY
|
| 47 |
+
# ==========================================================
|
| 48 |
+
SessionLocal = sessionmaker(
|
| 49 |
+
autocommit=False,
|
| 50 |
+
autoflush=False,
|
| 51 |
+
bind=engine,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
# ==========================================================
|
| 55 |
+
# BASE MODEL
|
| 56 |
+
# ==========================================================
|
| 57 |
+
Base = declarative_base()
|
| 58 |
+
|
| 59 |
+
# ==========================================================
|
| 60 |
+
# DEPENDENCY (FASTAPI)
|
| 61 |
+
# ==========================================================
|
| 62 |
+
def get_db():
|
| 63 |
+
"""
|
| 64 |
+
FastAPI dependency injection session.
|
| 65 |
+
Ensures connection cleanup even on crash.
|
| 66 |
+
"""
|
| 67 |
+
db = SessionLocal()
|
| 68 |
+
try:
|
| 69 |
+
yield db
|
| 70 |
+
finally:
|
| 71 |
+
db.close()
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# ==========================================================
|
| 75 |
+
# CONNECTION TEST (STARTUP SAFE)
|
| 76 |
+
# ==========================================================
|
| 77 |
+
def verify_database_connection():
|
| 78 |
+
"""
|
| 79 |
+
Validates database connectivity during startup.
|
| 80 |
+
Prevents silent runtime failures.
|
| 81 |
+
"""
|
| 82 |
+
try:
|
| 83 |
+
with engine.connect() as conn:
|
| 84 |
+
conn.execute("SELECT 1")
|
| 85 |
+
except OperationalError as e:
|
| 86 |
+
raise RuntimeError(
|
| 87 |
+
f"Database connection failed: {str(e)}"
|
| 88 |
+
)
|
auth/models.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uuid
|
| 2 |
+
from sqlalchemy import Column, String, Boolean, DateTime, func, Index
|
| 3 |
+
from sqlalchemy.dialects.postgresql import UUID
|
| 4 |
+
|
| 5 |
+
from auth.database import Base
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
# =========================================================
|
| 9 |
+
# USERS TABLE (CORE AUTH ENTITY)
|
| 10 |
+
# =========================================================
|
| 11 |
+
class User(Base):
|
| 12 |
+
__tablename__ = "users"
|
| 13 |
+
|
| 14 |
+
# -----------------------------
|
| 15 |
+
# Primary Key (UUID for Supabase compatibility)
|
| 16 |
+
# -----------------------------
|
| 17 |
+
id = Column(
|
| 18 |
+
UUID(as_uuid=True),
|
| 19 |
+
primary_key=True,
|
| 20 |
+
default=uuid.uuid4,
|
| 21 |
+
nullable=False,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
# -----------------------------
|
| 25 |
+
# Identity Fields
|
| 26 |
+
# -----------------------------
|
| 27 |
+
email = Column(String(255), unique=True, nullable=False, index=True)
|
| 28 |
+
username = Column(String(100), unique=True, nullable=True, index=True)
|
| 29 |
+
|
| 30 |
+
# -----------------------------
|
| 31 |
+
# Security Fields
|
| 32 |
+
# NOTE: stores hashed password only (never plaintext)
|
| 33 |
+
# -----------------------------
|
| 34 |
+
hashed_password = Column(String(255), nullable=False)
|
| 35 |
+
|
| 36 |
+
# -----------------------------
|
| 37 |
+
# Account State
|
| 38 |
+
# -----------------------------
|
| 39 |
+
is_active = Column(Boolean, default=True, nullable=False)
|
| 40 |
+
is_verified = Column(Boolean, default=False, nullable=False)
|
| 41 |
+
|
| 42 |
+
# -----------------------------
|
| 43 |
+
# Audit Fields
|
| 44 |
+
# -----------------------------
|
| 45 |
+
created_at = Column(
|
| 46 |
+
DateTime(timezone=True),
|
| 47 |
+
server_default=func.now(),
|
| 48 |
+
nullable=False,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
updated_at = Column(
|
| 52 |
+
DateTime(timezone=True),
|
| 53 |
+
server_default=func.now(),
|
| 54 |
+
onupdate=func.now(),
|
| 55 |
+
nullable=False,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# =========================================================
|
| 60 |
+
# OPTIONAL: API KEY TABLE (FOR AUTOMATION / N8N / WORKFLOWS)
|
| 61 |
+
# =========================================================
|
| 62 |
+
class ApiKey(Base):
|
| 63 |
+
__tablename__ = "api_keys"
|
| 64 |
+
|
| 65 |
+
id = Column(
|
| 66 |
+
UUID(as_uuid=True),
|
| 67 |
+
primary_key=True,
|
| 68 |
+
default=uuid.uuid4,
|
| 69 |
+
nullable=False,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
user_id = Column(
|
| 73 |
+
UUID(as_uuid=True),
|
| 74 |
+
nullable=False,
|
| 75 |
+
index=True,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
key_hash = Column(String(255), nullable=False, unique=True)
|
| 79 |
+
|
| 80 |
+
name = Column(String(120), nullable=True)
|
| 81 |
+
|
| 82 |
+
is_active = Column(Boolean, default=True, nullable=False)
|
| 83 |
+
|
| 84 |
+
created_at = Column(
|
| 85 |
+
DateTime(timezone=True),
|
| 86 |
+
server_default=func.now(),
|
| 87 |
+
nullable=False,
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
# =========================================================
|
| 92 |
+
# INDEXES (PERFORMANCE OPTIMIZATION)
|
| 93 |
+
# =========================================================
|
| 94 |
+
Index("idx_users_email", User.email)
|
| 95 |
+
Index("idx_users_username", User.username)
|
| 96 |
+
Index("idx_api_keys_user_id", ApiKey.user_id)
|
auth/routes.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Depends, HTTPException, status
|
| 2 |
+
from sqlalchemy.orm import Session
|
| 3 |
+
from sqlalchemy.exc import IntegrityError
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
|
| 6 |
+
from auth.database import get_db
|
| 7 |
+
from auth.models import User
|
| 8 |
+
from auth.schemas import (
|
| 9 |
+
SignupSchema,
|
| 10 |
+
LoginSchema,
|
| 11 |
+
TokenSchema,
|
| 12 |
+
)
|
| 13 |
+
from auth.security import (
|
| 14 |
+
hash_password,
|
| 15 |
+
verify_password,
|
| 16 |
+
create_access_token,
|
| 17 |
+
decode_token,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
router = APIRouter(
|
| 21 |
+
prefix="/api/auth",
|
| 22 |
+
tags=["Authentication"],
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# =========================================================
|
| 27 |
+
# SIGNUP
|
| 28 |
+
# =========================================================
|
| 29 |
+
|
| 30 |
+
@router.post("/signup", status_code=201)
|
| 31 |
+
def signup(
|
| 32 |
+
data: SignupSchema,
|
| 33 |
+
db: Session = Depends(get_db),
|
| 34 |
+
):
|
| 35 |
+
|
| 36 |
+
try:
|
| 37 |
+
|
| 38 |
+
# =================================================
|
| 39 |
+
# NORMALIZATION
|
| 40 |
+
# =================================================
|
| 41 |
+
|
| 42 |
+
email = data.email.strip().lower()
|
| 43 |
+
username = data.username.strip()
|
| 44 |
+
|
| 45 |
+
# =================================================
|
| 46 |
+
# EXISTING EMAIL CHECK
|
| 47 |
+
# =================================================
|
| 48 |
+
|
| 49 |
+
existing_email = (
|
| 50 |
+
db.query(User)
|
| 51 |
+
.filter(User.email == email)
|
| 52 |
+
.first()
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
if existing_email:
|
| 56 |
+
raise HTTPException(
|
| 57 |
+
status_code=status.HTTP_409_CONFLICT,
|
| 58 |
+
detail="Email already registered",
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
# =================================================
|
| 62 |
+
# EXISTING USERNAME CHECK
|
| 63 |
+
# =================================================
|
| 64 |
+
|
| 65 |
+
existing_username = (
|
| 66 |
+
db.query(User)
|
| 67 |
+
.filter(User.username == username)
|
| 68 |
+
.first()
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
if existing_username:
|
| 72 |
+
raise HTTPException(
|
| 73 |
+
status_code=status.HTTP_409_CONFLICT,
|
| 74 |
+
detail="Username already taken",
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
# =================================================
|
| 78 |
+
# HASH PASSWORD
|
| 79 |
+
# =================================================
|
| 80 |
+
|
| 81 |
+
password_hash = hash_password(data.password)
|
| 82 |
+
|
| 83 |
+
# =================================================
|
| 84 |
+
# CREATE USER
|
| 85 |
+
# =================================================
|
| 86 |
+
|
| 87 |
+
user = User(
|
| 88 |
+
email=email,
|
| 89 |
+
username=username,
|
| 90 |
+
password_hash=password_hash,
|
| 91 |
+
created_at=datetime.utcnow(),
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
db.add(user)
|
| 95 |
+
db.commit()
|
| 96 |
+
db.refresh(user)
|
| 97 |
+
|
| 98 |
+
# =================================================
|
| 99 |
+
# CREATE JWT
|
| 100 |
+
# =================================================
|
| 101 |
+
|
| 102 |
+
token = create_access_token(
|
| 103 |
+
{
|
| 104 |
+
"sub": str(user.id),
|
| 105 |
+
"email": user.email,
|
| 106 |
+
}
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
return {
|
| 110 |
+
"status": "success",
|
| 111 |
+
"message": "Account created successfully",
|
| 112 |
+
"token": token,
|
| 113 |
+
"user": {
|
| 114 |
+
"id": str(user.id),
|
| 115 |
+
"email": user.email,
|
| 116 |
+
"username": user.username,
|
| 117 |
+
"created_at": (
|
| 118 |
+
user.created_at.isoformat()
|
| 119 |
+
if user.created_at
|
| 120 |
+
else None
|
| 121 |
+
),
|
| 122 |
+
},
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
except HTTPException:
|
| 126 |
+
raise
|
| 127 |
+
|
| 128 |
+
except IntegrityError as e:
|
| 129 |
+
|
| 130 |
+
db.rollback()
|
| 131 |
+
|
| 132 |
+
raise HTTPException(
|
| 133 |
+
status_code=status.HTTP_409_CONFLICT,
|
| 134 |
+
detail="User already exists",
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
except ValueError as e:
|
| 138 |
+
|
| 139 |
+
db.rollback()
|
| 140 |
+
|
| 141 |
+
raise HTTPException(
|
| 142 |
+
status_code=status.HTTP_400_BAD_REQUEST,
|
| 143 |
+
detail=str(e),
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
except Exception as e:
|
| 147 |
+
|
| 148 |
+
db.rollback()
|
| 149 |
+
|
| 150 |
+
print("SIGNUP ERROR:", str(e))
|
| 151 |
+
|
| 152 |
+
raise HTTPException(
|
| 153 |
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
| 154 |
+
detail=f"Signup failed: {str(e)}",
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# =========================================================
|
| 159 |
+
# LOGIN
|
| 160 |
+
# =========================================================
|
| 161 |
+
|
| 162 |
+
@router.post("/login")
|
| 163 |
+
def login(
|
| 164 |
+
data: LoginSchema,
|
| 165 |
+
db: Session = Depends(get_db),
|
| 166 |
+
):
|
| 167 |
+
|
| 168 |
+
try:
|
| 169 |
+
|
| 170 |
+
email = data.email.strip().lower()
|
| 171 |
+
|
| 172 |
+
user = (
|
| 173 |
+
db.query(User)
|
| 174 |
+
.filter(User.email == email)
|
| 175 |
+
.first()
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
if not user:
|
| 179 |
+
|
| 180 |
+
raise HTTPException(
|
| 181 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 182 |
+
detail="Invalid email or password",
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
if not verify_password(
|
| 186 |
+
data.password,
|
| 187 |
+
user.password_hash,
|
| 188 |
+
):
|
| 189 |
+
|
| 190 |
+
raise HTTPException(
|
| 191 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 192 |
+
detail="Invalid email or password",
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
token = create_access_token(
|
| 196 |
+
{
|
| 197 |
+
"sub": str(user.id),
|
| 198 |
+
"email": user.email,
|
| 199 |
+
}
|
| 200 |
+
)
|
| 201 |
+
|
| 202 |
+
return {
|
| 203 |
+
"status": "success",
|
| 204 |
+
"message": "Login successful",
|
| 205 |
+
"token": token,
|
| 206 |
+
"user": {
|
| 207 |
+
"id": str(user.id),
|
| 208 |
+
"email": user.email,
|
| 209 |
+
"username": user.username,
|
| 210 |
+
"created_at": (
|
| 211 |
+
user.created_at.isoformat()
|
| 212 |
+
if user.created_at
|
| 213 |
+
else None
|
| 214 |
+
),
|
| 215 |
+
},
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
except HTTPException:
|
| 219 |
+
raise
|
| 220 |
+
|
| 221 |
+
except Exception as e:
|
| 222 |
+
|
| 223 |
+
print("LOGIN ERROR:", str(e))
|
| 224 |
+
|
| 225 |
+
raise HTTPException(
|
| 226 |
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
| 227 |
+
detail=f"Login failed: {str(e)}",
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# =========================================================
|
| 232 |
+
# VERIFY TOKEN
|
| 233 |
+
# =========================================================
|
| 234 |
+
|
| 235 |
+
@router.post("/verify")
|
| 236 |
+
def verify_token(
|
| 237 |
+
data: TokenSchema,
|
| 238 |
+
):
|
| 239 |
+
|
| 240 |
+
try:
|
| 241 |
+
|
| 242 |
+
payload = decode_token(data.token)
|
| 243 |
+
|
| 244 |
+
if not payload:
|
| 245 |
+
|
| 246 |
+
raise HTTPException(
|
| 247 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 248 |
+
detail="Invalid token",
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
return {
|
| 252 |
+
"valid": True,
|
| 253 |
+
"payload": payload,
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
except HTTPException:
|
| 257 |
+
raise
|
| 258 |
+
|
| 259 |
+
except Exception as e:
|
| 260 |
+
|
| 261 |
+
print("VERIFY ERROR:", str(e))
|
| 262 |
+
|
| 263 |
+
raise HTTPException(
|
| 264 |
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
| 265 |
+
detail=f"Token verification failed: {str(e)}",
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
# =========================================================
|
| 270 |
+
# REFRESH TOKEN
|
| 271 |
+
# =========================================================
|
| 272 |
+
|
| 273 |
+
@router.post("/refresh")
|
| 274 |
+
def refresh_token(
|
| 275 |
+
data: TokenSchema,
|
| 276 |
+
db: Session = Depends(get_db),
|
| 277 |
+
):
|
| 278 |
+
|
| 279 |
+
try:
|
| 280 |
+
|
| 281 |
+
payload = decode_token(data.token)
|
| 282 |
+
|
| 283 |
+
user_id = payload.get("sub")
|
| 284 |
+
|
| 285 |
+
if not user_id:
|
| 286 |
+
|
| 287 |
+
raise HTTPException(
|
| 288 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 289 |
+
detail="Invalid token payload",
|
| 290 |
+
)
|
| 291 |
+
|
| 292 |
+
user = (
|
| 293 |
+
db.query(User)
|
| 294 |
+
.filter(User.id == user_id)
|
| 295 |
+
.first()
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
if not user:
|
| 299 |
+
|
| 300 |
+
raise HTTPException(
|
| 301 |
+
status_code=status.HTTP_404_NOT_FOUND,
|
| 302 |
+
detail="User not found",
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
new_token = create_access_token(
|
| 306 |
+
{
|
| 307 |
+
"sub": str(user.id),
|
| 308 |
+
"email": user.email,
|
| 309 |
+
}
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
return {
|
| 313 |
+
"status": "success",
|
| 314 |
+
"token": new_token,
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
except HTTPException:
|
| 318 |
+
raise
|
| 319 |
+
|
| 320 |
+
except Exception as e:
|
| 321 |
+
|
| 322 |
+
print("REFRESH ERROR:", str(e))
|
| 323 |
+
|
| 324 |
+
raise HTTPException(
|
| 325 |
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
| 326 |
+
detail=f"Refresh failed: {str(e)}",
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
# =========================================================
|
| 331 |
+
# LOGOUT
|
| 332 |
+
# =========================================================
|
| 333 |
+
|
| 334 |
+
@router.post("/logout")
|
| 335 |
+
def logout():
|
| 336 |
+
|
| 337 |
+
return {
|
| 338 |
+
"status": "success",
|
| 339 |
+
"message": "Logout successful",
|
| 340 |
+
}
|
auth/schemas.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel, EmailStr, Field, ConfigDict
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
# =========================================================
|
| 5 |
+
# BASE CONFIG (STRICT MODE → PREVENTS SILENT DATA COERCION)
|
| 6 |
+
# =========================================================
|
| 7 |
+
class StrictSchema(BaseModel):
|
| 8 |
+
model_config = ConfigDict(
|
| 9 |
+
strict=True,
|
| 10 |
+
extra="forbid",
|
| 11 |
+
validate_assignment=True,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
# =========================================================
|
| 16 |
+
# SIGNUP SCHEMA
|
| 17 |
+
# =========================================================
|
| 18 |
+
class SignupSchema(StrictSchema):
|
| 19 |
+
email: EmailStr
|
| 20 |
+
username: str = Field(
|
| 21 |
+
min_length=3,
|
| 22 |
+
max_length=32,
|
| 23 |
+
pattern=r"^[a-zA-Z0-9_]+$"
|
| 24 |
+
)
|
| 25 |
+
password: str = Field(
|
| 26 |
+
min_length=8,
|
| 27 |
+
max_length=72,
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# =========================================================
|
| 32 |
+
# LOGIN SCHEMA
|
| 33 |
+
# =========================================================
|
| 34 |
+
class LoginSchema(StrictSchema):
|
| 35 |
+
email: EmailStr
|
| 36 |
+
password: str = Field(
|
| 37 |
+
min_length=1,
|
| 38 |
+
max_length=72,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# =========================================================
|
| 43 |
+
# TOKEN REQUEST SCHEMA
|
| 44 |
+
# =========================================================
|
| 45 |
+
class TokenSchema(StrictSchema):
|
| 46 |
+
token: str = Field(
|
| 47 |
+
min_length=10,
|
| 48 |
+
max_length=2048
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# =========================================================
|
| 53 |
+
# USER RESPONSE SCHEMA (SAFE OUTPUT MODEL)
|
| 54 |
+
# =========================================================
|
| 55 |
+
class UserOutSchema(StrictSchema):
|
| 56 |
+
id: int
|
| 57 |
+
email: EmailStr
|
| 58 |
+
username: str
|
| 59 |
+
created_at: str
|
auth/security.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from datetime import datetime, timedelta, timezone
|
| 3 |
+
from typing import Optional, Dict, Any
|
| 4 |
+
|
| 5 |
+
from jose import jwt, JWTError
|
| 6 |
+
from passlib.context import CryptContext
|
| 7 |
+
from fastapi import HTTPException, status, Depends
|
| 8 |
+
from fastapi.security import OAuth2PasswordBearer
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
# =========================================================
|
| 12 |
+
# ENV CONFIG
|
| 13 |
+
# =========================================================
|
| 14 |
+
|
| 15 |
+
SECRET_KEY = os.getenv("SECRET_KEY")
|
| 16 |
+
|
| 17 |
+
if not SECRET_KEY:
|
| 18 |
+
raise RuntimeError("SECRET_KEY environment variable missing")
|
| 19 |
+
|
| 20 |
+
ALGORITHM = os.getenv("ALGORITHM", "HS256")
|
| 21 |
+
|
| 22 |
+
ACCESS_TOKEN_EXPIRE_MINUTES = int(
|
| 23 |
+
os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "1440")
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# =========================================================
|
| 28 |
+
# PASSWORD HASHING
|
| 29 |
+
# =========================================================
|
| 30 |
+
# IMPORTANT:
|
| 31 |
+
# pbkdf2_sha256 avoids:
|
| 32 |
+
# - bcrypt crashes
|
| 33 |
+
# - bcrypt native dependency issues
|
| 34 |
+
# - 72-byte limits
|
| 35 |
+
# - passlib backend bugs
|
| 36 |
+
# =========================================================
|
| 37 |
+
|
| 38 |
+
pwd_context = CryptContext(
|
| 39 |
+
schemes=["pbkdf2_sha256"],
|
| 40 |
+
deprecated="auto",
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def hash_password(password: str) -> str:
|
| 45 |
+
|
| 46 |
+
if not password:
|
| 47 |
+
raise ValueError("Password required")
|
| 48 |
+
|
| 49 |
+
if len(password) < 8:
|
| 50 |
+
raise ValueError("Password too short")
|
| 51 |
+
|
| 52 |
+
return pwd_context.hash(password)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def verify_password(
|
| 56 |
+
plain_password: str,
|
| 57 |
+
hashed_password: str,
|
| 58 |
+
) -> bool:
|
| 59 |
+
|
| 60 |
+
try:
|
| 61 |
+
return pwd_context.verify(
|
| 62 |
+
plain_password,
|
| 63 |
+
hashed_password,
|
| 64 |
+
)
|
| 65 |
+
except Exception:
|
| 66 |
+
return False
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# =========================================================
|
| 70 |
+
# JWT
|
| 71 |
+
# =========================================================
|
| 72 |
+
|
| 73 |
+
def create_access_token(
|
| 74 |
+
data: Dict[str, Any],
|
| 75 |
+
expires_delta: Optional[timedelta] = None,
|
| 76 |
+
) -> str:
|
| 77 |
+
|
| 78 |
+
to_encode = data.copy()
|
| 79 |
+
|
| 80 |
+
expire = datetime.now(timezone.utc) + (
|
| 81 |
+
expires_delta
|
| 82 |
+
if expires_delta
|
| 83 |
+
else timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
to_encode.update({"exp": expire})
|
| 87 |
+
|
| 88 |
+
return jwt.encode(
|
| 89 |
+
to_encode,
|
| 90 |
+
SECRET_KEY,
|
| 91 |
+
algorithm=ALGORITHM,
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def decode_token(token: str):
|
| 96 |
+
|
| 97 |
+
try:
|
| 98 |
+
return jwt.decode(
|
| 99 |
+
token,
|
| 100 |
+
SECRET_KEY,
|
| 101 |
+
algorithms=[ALGORITHM],
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
except JWTError:
|
| 105 |
+
|
| 106 |
+
raise HTTPException(
|
| 107 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 108 |
+
detail="Invalid token",
|
| 109 |
+
headers={"WWW-Authenticate": "Bearer"},
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
# =========================================================
|
| 114 |
+
# AUTH DEPENDENCY
|
| 115 |
+
# =========================================================
|
| 116 |
+
|
| 117 |
+
oauth2_scheme = OAuth2PasswordBearer(
|
| 118 |
+
tokenUrl="/api/auth/login"
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def get_current_user(
|
| 123 |
+
token: str = Depends(oauth2_scheme)
|
| 124 |
+
):
|
| 125 |
+
|
| 126 |
+
payload = decode_token(token)
|
| 127 |
+
|
| 128 |
+
user_id = payload.get("sub")
|
| 129 |
+
|
| 130 |
+
if not user_id:
|
| 131 |
+
|
| 132 |
+
raise HTTPException(
|
| 133 |
+
status_code=401,
|
| 134 |
+
detail="Invalid authentication",
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
return payload
|
core/builders/api_builder.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, UploadFile, File, Form, HTTPException
|
| 2 |
+
from fastapi.responses import JSONResponse, FileResponse
|
| 3 |
+
from typing import Optional
|
| 4 |
+
import tempfile
|
| 5 |
+
import shutil
|
| 6 |
+
import os
|
| 7 |
+
|
| 8 |
+
from core.registry.loader import get_tasks
|
| 9 |
+
from core.execution.executor import execute_task
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
# =====================================================
|
| 13 |
+
# ROUTER BUILDER
|
| 14 |
+
# =====================================================
|
| 15 |
+
|
| 16 |
+
def build_api_router() -> APIRouter:
|
| 17 |
+
"""
|
| 18 |
+
Dynamically builds API routes from TASK REGISTRY.
|
| 19 |
+
|
| 20 |
+
Generated endpoints:
|
| 21 |
+
|
| 22 |
+
POST /execute/{task}
|
| 23 |
+
GET /tasks
|
| 24 |
+
GET /health
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
router = APIRouter(tags=["API"])
|
| 28 |
+
|
| 29 |
+
# =================================================
|
| 30 |
+
# EXECUTE TASK
|
| 31 |
+
# =================================================
|
| 32 |
+
@router.post("/execute/{task_name}")
|
| 33 |
+
async def execute(
|
| 34 |
+
task_name: str,
|
| 35 |
+
file: Optional[UploadFile] = File(None),
|
| 36 |
+
url_input: Optional[str] = Form(None),
|
| 37 |
+
):
|
| 38 |
+
"""
|
| 39 |
+
Universal execution endpoint.
|
| 40 |
+
Accepts:
|
| 41 |
+
- file upload
|
| 42 |
+
- url_input
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
tasks = get_tasks()
|
| 46 |
+
|
| 47 |
+
if task_name not in tasks:
|
| 48 |
+
raise HTTPException(
|
| 49 |
+
status_code=404,
|
| 50 |
+
detail=f"Task '{task_name}' not found",
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
temp_path = None
|
| 54 |
+
|
| 55 |
+
try:
|
| 56 |
+
# -----------------------------------------
|
| 57 |
+
# SAVE UPLOADED FILE
|
| 58 |
+
# -----------------------------------------
|
| 59 |
+
if file:
|
| 60 |
+
suffix = os.path.splitext(file.filename)[1]
|
| 61 |
+
|
| 62 |
+
with tempfile.NamedTemporaryFile(
|
| 63 |
+
delete=False,
|
| 64 |
+
suffix=suffix,
|
| 65 |
+
) as tmp:
|
| 66 |
+
shutil.copyfileobj(file.file, tmp)
|
| 67 |
+
temp_path = tmp.name
|
| 68 |
+
|
| 69 |
+
# -----------------------------------------
|
| 70 |
+
# BUILD INPUT PAYLOAD
|
| 71 |
+
# -----------------------------------------
|
| 72 |
+
payload = {
|
| 73 |
+
"file_path": temp_path,
|
| 74 |
+
"url_input": url_input,
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
# -----------------------------------------
|
| 78 |
+
# EXECUTE TASK
|
| 79 |
+
# -----------------------------------------
|
| 80 |
+
result = await execute_task(task_name, payload)
|
| 81 |
+
|
| 82 |
+
# -----------------------------------------
|
| 83 |
+
# FILE RESPONSE
|
| 84 |
+
# -----------------------------------------
|
| 85 |
+
if isinstance(result, dict) and result.get("file"):
|
| 86 |
+
output_file = result["file"]
|
| 87 |
+
|
| 88 |
+
if os.path.exists(output_file):
|
| 89 |
+
return FileResponse(
|
| 90 |
+
output_file,
|
| 91 |
+
filename=os.path.basename(output_file),
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
# -----------------------------------------
|
| 95 |
+
# JSON RESPONSE
|
| 96 |
+
# -----------------------------------------
|
| 97 |
+
return JSONResponse(result)
|
| 98 |
+
|
| 99 |
+
except Exception as e:
|
| 100 |
+
raise HTTPException(
|
| 101 |
+
status_code=500,
|
| 102 |
+
detail=str(e),
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
finally:
|
| 106 |
+
# -----------------------------------------
|
| 107 |
+
# CLEANUP TEMP FILE
|
| 108 |
+
# -----------------------------------------
|
| 109 |
+
if temp_path and os.path.exists(temp_path):
|
| 110 |
+
os.unlink(temp_path)
|
| 111 |
+
|
| 112 |
+
# =================================================
|
| 113 |
+
# TASK LIST
|
| 114 |
+
# =================================================
|
| 115 |
+
@router.get("/tasks")
|
| 116 |
+
async def list_tasks():
|
| 117 |
+
"""
|
| 118 |
+
Returns registry tasks.
|
| 119 |
+
Used by UI and Docs builder.
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
tasks = get_tasks()
|
| 123 |
+
|
| 124 |
+
return {
|
| 125 |
+
name: {
|
| 126 |
+
"category": getattr(t, "category", "general"),
|
| 127 |
+
"description": getattr(t, "description", ""),
|
| 128 |
+
}
|
| 129 |
+
for name, t in tasks.items()
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
# =================================================
|
| 133 |
+
# HEALTH CHECK
|
| 134 |
+
# =================================================
|
| 135 |
+
@router.get("/health")
|
| 136 |
+
async def health():
|
| 137 |
+
return {
|
| 138 |
+
"status": "ok",
|
| 139 |
+
"tasks_loaded": len(get_tasks()),
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
return router
|
core/builders/docs_builder.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from fastapi.responses import JSONResponse
|
| 3 |
+
from typing import Dict, Any, List
|
| 4 |
+
|
| 5 |
+
from core.registry.tasks_registry import TASKS
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
# =====================================================
|
| 9 |
+
# INTERNAL HELPERS
|
| 10 |
+
# =====================================================
|
| 11 |
+
|
| 12 |
+
def _serialize_task(task) -> Dict[str, Any]:
|
| 13 |
+
"""
|
| 14 |
+
Convert TaskDefinition → API-safe metadata.
|
| 15 |
+
Must NOT import task modules.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
return {
|
| 19 |
+
"name": getattr(task, "name", None),
|
| 20 |
+
"description": getattr(task, "description", ""),
|
| 21 |
+
"category": getattr(task, "category", "general"),
|
| 22 |
+
"module": getattr(task, "module", None),
|
| 23 |
+
"callable": getattr(task, "callable_name", "run"),
|
| 24 |
+
"async": getattr(task, "async_task", False),
|
| 25 |
+
"enabled": getattr(task, "enabled", True),
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _group_tasks(tasks: List[Any]) -> Dict[str, List[Dict[str, Any]]]:
|
| 30 |
+
"""
|
| 31 |
+
Groups tasks by category for UI rendering.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
grouped: Dict[str, List[Dict[str, Any]]] = {}
|
| 35 |
+
|
| 36 |
+
for task in tasks:
|
| 37 |
+
category = getattr(task, "category", "general")
|
| 38 |
+
grouped.setdefault(category, []).append(_serialize_task(task))
|
| 39 |
+
|
| 40 |
+
return grouped
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# =====================================================
|
| 44 |
+
# DOCS ROUTER BUILDER (V11)
|
| 45 |
+
# =====================================================
|
| 46 |
+
|
| 47 |
+
def build_docs_router() -> APIRouter:
|
| 48 |
+
"""
|
| 49 |
+
Builds dynamic documentation endpoints.
|
| 50 |
+
|
| 51 |
+
Provides:
|
| 52 |
+
/docs/tasks → flat task list
|
| 53 |
+
/docs/catalog → grouped tasks
|
| 54 |
+
/docs/health → docs status
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
router = APIRouter(
|
| 58 |
+
prefix="/docs",
|
| 59 |
+
tags=["Documentation"],
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
# -------------------------------------------------
|
| 63 |
+
# List all tasks
|
| 64 |
+
# -------------------------------------------------
|
| 65 |
+
@router.get("/tasks")
|
| 66 |
+
async def list_tasks():
|
| 67 |
+
return JSONResponse(
|
| 68 |
+
[_serialize_task(task) for task in TASKS]
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
# -------------------------------------------------
|
| 72 |
+
# Categorized task catalog
|
| 73 |
+
# -------------------------------------------------
|
| 74 |
+
@router.get("/catalog")
|
| 75 |
+
async def task_catalog():
|
| 76 |
+
return JSONResponse(_group_tasks(TASKS))
|
| 77 |
+
|
| 78 |
+
# -------------------------------------------------
|
| 79 |
+
# Docs health endpoint
|
| 80 |
+
# -------------------------------------------------
|
| 81 |
+
@router.get("/health")
|
| 82 |
+
async def docs_health():
|
| 83 |
+
return {
|
| 84 |
+
"status": "ok",
|
| 85 |
+
"service": "docs",
|
| 86 |
+
"tasks_registered": len(TASKS),
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
return router
|
core/builders/ui_builder.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from fastapi.responses import HTMLResponse, FileResponse
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Dict, List
|
| 5 |
+
|
| 6 |
+
from core.registry.tasks_registry import TASKS
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
# =====================================================
|
| 10 |
+
# CONFIG
|
| 11 |
+
# =====================================================
|
| 12 |
+
|
| 13 |
+
BASE_DIR = Path(__file__).resolve().parents[2]
|
| 14 |
+
UI_DIR = BASE_DIR / "ui"
|
| 15 |
+
INDEX_FILE = UI_DIR / "index.html"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# =====================================================
|
| 19 |
+
# TASK GROUPING
|
| 20 |
+
# =====================================================
|
| 21 |
+
|
| 22 |
+
def group_tasks() -> Dict[str, List[dict]]:
|
| 23 |
+
"""
|
| 24 |
+
Converts registry → categorized UI structure.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
categories: Dict[str, List[dict]] = {}
|
| 28 |
+
|
| 29 |
+
for task in TASKS:
|
| 30 |
+
if not getattr(task, "enabled", True):
|
| 31 |
+
continue
|
| 32 |
+
|
| 33 |
+
category = getattr(task, "category", "general")
|
| 34 |
+
|
| 35 |
+
categories.setdefault(category, []).append(
|
| 36 |
+
{
|
| 37 |
+
"name": task.name,
|
| 38 |
+
"description": getattr(task, "description", ""),
|
| 39 |
+
}
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
return categories
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# =====================================================
|
| 46 |
+
# ROUTER BUILDER
|
| 47 |
+
# =====================================================
|
| 48 |
+
|
| 49 |
+
def build_ui_router() -> APIRouter:
|
| 50 |
+
"""
|
| 51 |
+
Serves UI + dynamic UI metadata.
|
| 52 |
+
|
| 53 |
+
Endpoints:
|
| 54 |
+
/
|
| 55 |
+
/ui
|
| 56 |
+
/ui/tasks
|
| 57 |
+
/ui/health
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
router = APIRouter(tags=["UI"])
|
| 61 |
+
|
| 62 |
+
# -------------------------------------------------
|
| 63 |
+
# MAIN UI
|
| 64 |
+
# -------------------------------------------------
|
| 65 |
+
@router.get("/", response_class=HTMLResponse)
|
| 66 |
+
async def serve_root():
|
| 67 |
+
"""
|
| 68 |
+
Serves index.html
|
| 69 |
+
"""
|
| 70 |
+
|
| 71 |
+
if not INDEX_FILE.exists():
|
| 72 |
+
return HTMLResponse(
|
| 73 |
+
"<h1>Basyx UI Missing</h1>",
|
| 74 |
+
status_code=500,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
return FileResponse(INDEX_FILE)
|
| 78 |
+
|
| 79 |
+
# -------------------------------------------------
|
| 80 |
+
# Explicit UI route
|
| 81 |
+
# -------------------------------------------------
|
| 82 |
+
@router.get("/ui", response_class=HTMLResponse)
|
| 83 |
+
async def serve_ui():
|
| 84 |
+
return await serve_root()
|
| 85 |
+
|
| 86 |
+
# -------------------------------------------------
|
| 87 |
+
# UI TASK CATALOG
|
| 88 |
+
# -------------------------------------------------
|
| 89 |
+
@router.get("/ui/tasks")
|
| 90 |
+
async def ui_tasks():
|
| 91 |
+
"""
|
| 92 |
+
UI fetches this to build sidebar dynamically.
|
| 93 |
+
"""
|
| 94 |
+
return group_tasks()
|
| 95 |
+
|
| 96 |
+
# -------------------------------------------------
|
| 97 |
+
# HEALTH
|
| 98 |
+
# -------------------------------------------------
|
| 99 |
+
@router.get("/ui/health")
|
| 100 |
+
async def ui_health():
|
| 101 |
+
return {
|
| 102 |
+
"status": "ok",
|
| 103 |
+
"ui": "active",
|
| 104 |
+
"tasks": len(TASKS),
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
return router
|
core/execution/context.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
V11 Execution Context
|
| 3 |
+
---------------------
|
| 4 |
+
|
| 5 |
+
Central runtime object passed into every task.
|
| 6 |
+
|
| 7 |
+
Responsibilities:
|
| 8 |
+
- Hold inputs
|
| 9 |
+
- Share memory between tasks
|
| 10 |
+
- Store outputs
|
| 11 |
+
- Track execution metadata
|
| 12 |
+
- Provide filesystem helpers
|
| 13 |
+
- Provide logging helpers
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import os
|
| 19 |
+
import uuid
|
| 20 |
+
import tempfile
|
| 21 |
+
from typing import Any, Dict, Optional
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
# =========================================================
|
| 25 |
+
# Context Object
|
| 26 |
+
# =========================================================
|
| 27 |
+
|
| 28 |
+
class ExecutionContext:
|
| 29 |
+
"""
|
| 30 |
+
Standard runtime context used by ALL tasks.
|
| 31 |
+
|
| 32 |
+
Every task receives:
|
| 33 |
+
async def run(ctx: ExecutionContext)
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
# -----------------------------------------------------
|
| 37 |
+
# INIT
|
| 38 |
+
# -----------------------------------------------------
|
| 39 |
+
def __init__(
|
| 40 |
+
self,
|
| 41 |
+
task_name: str,
|
| 42 |
+
inputs: Optional[Dict[str, Any]] = None,
|
| 43 |
+
workspace: Optional[str] = None,
|
| 44 |
+
):
|
| 45 |
+
|
| 46 |
+
self.task_name = task_name
|
| 47 |
+
self.job_id = str(uuid.uuid4())
|
| 48 |
+
|
| 49 |
+
self.inputs: Dict[str, Any] = inputs or {}
|
| 50 |
+
self.outputs: Dict[str, Any] = {}
|
| 51 |
+
self.memory: Dict[str, Any] = {}
|
| 52 |
+
|
| 53 |
+
self.status: str = "created"
|
| 54 |
+
self.error: Optional[str] = None
|
| 55 |
+
|
| 56 |
+
self.workspace = workspace or self._create_workspace()
|
| 57 |
+
|
| 58 |
+
# -----------------------------------------------------
|
| 59 |
+
# WORKSPACE
|
| 60 |
+
# -----------------------------------------------------
|
| 61 |
+
def _create_workspace(self) -> str:
|
| 62 |
+
path = tempfile.mkdtemp(prefix="basyx_job_")
|
| 63 |
+
return path
|
| 64 |
+
|
| 65 |
+
def path(self, filename: str) -> str:
|
| 66 |
+
"""
|
| 67 |
+
Safe workspace path helper
|
| 68 |
+
"""
|
| 69 |
+
return os.path.join(self.workspace, filename)
|
| 70 |
+
|
| 71 |
+
# -----------------------------------------------------
|
| 72 |
+
# INPUT HELPERS
|
| 73 |
+
# -----------------------------------------------------
|
| 74 |
+
def get(self, key: str, default=None):
|
| 75 |
+
return self.inputs.get(key, default)
|
| 76 |
+
|
| 77 |
+
def require(self, key: str):
|
| 78 |
+
if key not in self.inputs:
|
| 79 |
+
raise ValueError(f"Missing required input: {key}")
|
| 80 |
+
return self.inputs[key]
|
| 81 |
+
|
| 82 |
+
# -----------------------------------------------------
|
| 83 |
+
# OUTPUT HELPERS
|
| 84 |
+
# -----------------------------------------------------
|
| 85 |
+
def set_output(self, key: str, value: Any):
|
| 86 |
+
self.outputs[key] = value
|
| 87 |
+
|
| 88 |
+
def result(self) -> Dict[str, Any]:
|
| 89 |
+
return {
|
| 90 |
+
"job_id": self.job_id,
|
| 91 |
+
"task": self.task_name,
|
| 92 |
+
"status": self.status,
|
| 93 |
+
"outputs": self.outputs,
|
| 94 |
+
"error": self.error,
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
# -----------------------------------------------------
|
| 98 |
+
# MEMORY (cross-task sharing)
|
| 99 |
+
# -----------------------------------------------------
|
| 100 |
+
def remember(self, key: str, value: Any):
|
| 101 |
+
"""
|
| 102 |
+
Save value for downstream tasks.
|
| 103 |
+
"""
|
| 104 |
+
self.memory[key] = value
|
| 105 |
+
|
| 106 |
+
def recall(self, key: str, default=None):
|
| 107 |
+
return self.memory.get(key, default)
|
| 108 |
+
|
| 109 |
+
# -----------------------------------------------------
|
| 110 |
+
# STATUS MANAGEMENT
|
| 111 |
+
# -----------------------------------------------------
|
| 112 |
+
def mark_running(self):
|
| 113 |
+
self.status = "running"
|
| 114 |
+
|
| 115 |
+
def mark_complete(self):
|
| 116 |
+
self.status = "completed"
|
| 117 |
+
|
| 118 |
+
def mark_failed(self, error: Exception | str):
|
| 119 |
+
self.status = "failed"
|
| 120 |
+
self.error = str(error)
|
| 121 |
+
|
| 122 |
+
# -----------------------------------------------------
|
| 123 |
+
# LOGGING
|
| 124 |
+
# -----------------------------------------------------
|
| 125 |
+
def log(self, message: str):
|
| 126 |
+
print(f"[{self.task_name} | {self.job_id}] {message}")
|
| 127 |
+
|
| 128 |
+
# -----------------------------------------------------
|
| 129 |
+
# SERIALIZATION
|
| 130 |
+
# -----------------------------------------------------
|
| 131 |
+
def to_dict(self):
|
| 132 |
+
return {
|
| 133 |
+
"job_id": self.job_id,
|
| 134 |
+
"task_name": self.task_name,
|
| 135 |
+
"inputs": self.inputs,
|
| 136 |
+
"outputs": self.outputs,
|
| 137 |
+
"memory": self.memory,
|
| 138 |
+
"status": self.status,
|
| 139 |
+
"error": self.error,
|
| 140 |
+
"workspace": self.workspace,
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
# =========================================================
|
| 145 |
+
# Context Factory
|
| 146 |
+
# =========================================================
|
| 147 |
+
|
| 148 |
+
def create_context(task_name: str, inputs: Dict[str, Any]) -> ExecutionContext:
|
| 149 |
+
"""
|
| 150 |
+
Standardized factory used by executor.
|
| 151 |
+
"""
|
| 152 |
+
return ExecutionContext(
|
| 153 |
+
task_name=task_name,
|
| 154 |
+
inputs=inputs,
|
| 155 |
+
)
|
core/execution/executor.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
BASYX V11 EXECUTOR
|
| 3 |
+
------------------
|
| 4 |
+
|
| 5 |
+
Central task execution engine.
|
| 6 |
+
|
| 7 |
+
Responsibilities:
|
| 8 |
+
- Load task from registry
|
| 9 |
+
- Create execution context
|
| 10 |
+
- Execute task safely
|
| 11 |
+
- Capture outputs
|
| 12 |
+
- Handle failures
|
| 13 |
+
- Support chaining
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import asyncio
|
| 19 |
+
import inspect
|
| 20 |
+
from typing import Dict, Any, List
|
| 21 |
+
|
| 22 |
+
from core.execution.context import create_context, ExecutionContext
|
| 23 |
+
from core.registry.loader import get_task_map
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# =========================================================
|
| 27 |
+
# TASK CACHE
|
| 28 |
+
# =========================================================
|
| 29 |
+
|
| 30 |
+
TASK_MAP = get_task_map()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# =========================================================
|
| 34 |
+
# INTERNAL EXECUTION
|
| 35 |
+
# =========================================================
|
| 36 |
+
|
| 37 |
+
async def _run_task(
|
| 38 |
+
task_name: str,
|
| 39 |
+
inputs: Dict[str, Any],
|
| 40 |
+
) -> Dict[str, Any]:
|
| 41 |
+
"""
|
| 42 |
+
Execute a single task safely.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
if task_name not in TASK_MAP:
|
| 46 |
+
raise ValueError(f"Unknown task: {task_name}")
|
| 47 |
+
|
| 48 |
+
task = TASK_MAP[task_name]
|
| 49 |
+
|
| 50 |
+
ctx: ExecutionContext = create_context(
|
| 51 |
+
task_name=task_name,
|
| 52 |
+
inputs=inputs,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
ctx.mark_running()
|
| 56 |
+
ctx.log("Starting task")
|
| 57 |
+
|
| 58 |
+
try:
|
| 59 |
+
|
| 60 |
+
# ---------------------------------------------
|
| 61 |
+
# Execute task
|
| 62 |
+
# ---------------------------------------------
|
| 63 |
+
result = task.run
|
| 64 |
+
|
| 65 |
+
if inspect.iscoroutinefunction(result):
|
| 66 |
+
await result(ctx)
|
| 67 |
+
else:
|
| 68 |
+
await asyncio.to_thread(result, ctx)
|
| 69 |
+
|
| 70 |
+
ctx.mark_complete()
|
| 71 |
+
ctx.log("Task completed")
|
| 72 |
+
|
| 73 |
+
except Exception as e:
|
| 74 |
+
ctx.mark_failed(e)
|
| 75 |
+
ctx.log(f"Task failed: {e}")
|
| 76 |
+
|
| 77 |
+
return ctx.result()
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
# =========================================================
|
| 81 |
+
# PUBLIC EXECUTOR
|
| 82 |
+
# =========================================================
|
| 83 |
+
|
| 84 |
+
async def execute_task(
|
| 85 |
+
task_name: str,
|
| 86 |
+
inputs: Dict[str, Any],
|
| 87 |
+
) -> Dict[str, Any]:
|
| 88 |
+
"""
|
| 89 |
+
Main entrypoint used by API + UI.
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
return await _run_task(task_name, inputs)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# =========================================================
|
| 96 |
+
# PIPELINE EXECUTION (CHAINED TASKS)
|
| 97 |
+
# =========================================================
|
| 98 |
+
|
| 99 |
+
async def execute_pipeline(
|
| 100 |
+
tasks: List[Dict[str, Any]]
|
| 101 |
+
) -> List[Dict[str, Any]]:
|
| 102 |
+
"""
|
| 103 |
+
Execute tasks sequentially.
|
| 104 |
+
|
| 105 |
+
Example:
|
| 106 |
+
[
|
| 107 |
+
{"task": "transcribe", "inputs": {...}},
|
| 108 |
+
{"task": "subtitles"},
|
| 109 |
+
{"task": "render"}
|
| 110 |
+
]
|
| 111 |
+
"""
|
| 112 |
+
|
| 113 |
+
results = []
|
| 114 |
+
shared_memory = {}
|
| 115 |
+
|
| 116 |
+
for step in tasks:
|
| 117 |
+
|
| 118 |
+
name = step["task"]
|
| 119 |
+
inputs = step.get("inputs", {})
|
| 120 |
+
|
| 121 |
+
# Inject memory from previous step
|
| 122 |
+
inputs["memory"] = shared_memory
|
| 123 |
+
|
| 124 |
+
result = await _run_task(name, inputs)
|
| 125 |
+
|
| 126 |
+
results.append(result)
|
| 127 |
+
|
| 128 |
+
if result["status"] != "completed":
|
| 129 |
+
break
|
| 130 |
+
|
| 131 |
+
# propagate outputs
|
| 132 |
+
shared_memory.update(result.get("outputs", {}))
|
| 133 |
+
|
| 134 |
+
return results
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# =========================================================
|
| 138 |
+
# PARALLEL EXECUTION
|
| 139 |
+
# =========================================================
|
| 140 |
+
|
| 141 |
+
async def execute_parallel(
|
| 142 |
+
tasks: List[Dict[str, Any]]
|
| 143 |
+
) -> List[Dict[str, Any]]:
|
| 144 |
+
"""
|
| 145 |
+
Run multiple tasks concurrently.
|
| 146 |
+
"""
|
| 147 |
+
|
| 148 |
+
coroutines = [
|
| 149 |
+
_run_task(t["task"], t.get("inputs", {}))
|
| 150 |
+
for t in tasks
|
| 151 |
+
]
|
| 152 |
+
|
| 153 |
+
return await asyncio.gather(*coroutines)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
# =========================================================
|
| 157 |
+
# REGISTRY HOT RELOAD (DEV MODE)
|
| 158 |
+
# =========================================================
|
| 159 |
+
|
| 160 |
+
def reload_tasks():
|
| 161 |
+
"""
|
| 162 |
+
Reload registry without restarting server.
|
| 163 |
+
Useful during development.
|
| 164 |
+
"""
|
| 165 |
+
global TASK_MAP
|
| 166 |
+
TASK_MAP = get_task_map()
|
core/registry/loader.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from core.registry.tasks_registry import TASKS
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
# ------------------------------------------------
|
| 5 |
+
# Return categorized tasks (UI Builder uses this)
|
| 6 |
+
# ------------------------------------------------
|
| 7 |
+
def get_tasks():
|
| 8 |
+
return TASKS
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
# ------------------------------------------------
|
| 12 |
+
# Flatten tasks across categories
|
| 13 |
+
# ------------------------------------------------
|
| 14 |
+
def get_all_tasks():
|
| 15 |
+
|
| 16 |
+
tasks = []
|
| 17 |
+
|
| 18 |
+
for category, items in TASKS.items():
|
| 19 |
+
for task in items:
|
| 20 |
+
task_copy = dict(task)
|
| 21 |
+
task_copy["category"] = category
|
| 22 |
+
tasks.append(task_copy)
|
| 23 |
+
|
| 24 |
+
return tasks
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# ------------------------------------------------
|
| 28 |
+
# Execution map
|
| 29 |
+
# id -> callable
|
| 30 |
+
# ------------------------------------------------
|
| 31 |
+
def get_task_map():
|
| 32 |
+
|
| 33 |
+
task_map = {}
|
| 34 |
+
|
| 35 |
+
for category, items in TASKS.items():
|
| 36 |
+
for task in items:
|
| 37 |
+
|
| 38 |
+
if "handler" not in task:
|
| 39 |
+
raise RuntimeError(
|
| 40 |
+
f"Task '{task['id']}' missing handler"
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
task_map[task["id"]] = task["handler"]
|
| 44 |
+
|
| 45 |
+
return task_map
|
core/registry/task_model.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel, Field
|
| 2 |
+
from typing import Callable, Dict, Any
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class TaskDefinition(BaseModel):
|
| 6 |
+
name: str
|
| 7 |
+
category: str
|
| 8 |
+
description: str
|
| 9 |
+
|
| 10 |
+
handler: Callable
|
| 11 |
+
|
| 12 |
+
inputs: Dict[str, str] = Field(default_factory=dict)
|
| 13 |
+
outputs: Dict[str, str] = Field(default_factory=dict)
|
| 14 |
+
|
| 15 |
+
ui_schema: Dict[str, Any] = Field(default_factory=dict)
|
| 16 |
+
|
| 17 |
+
autonomous: bool = True
|
| 18 |
+
|
| 19 |
+
class Config:
|
| 20 |
+
arbitrary_types_allowed = True
|
core/registry/tasks_registry.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
BASYX V11 — Dynamic Task Registry
|
| 3 |
+
---------------------------------
|
| 4 |
+
|
| 5 |
+
Single Source Of Truth for:
|
| 6 |
+
|
| 7 |
+
• UI generation
|
| 8 |
+
• API routes
|
| 9 |
+
• Executor routing
|
| 10 |
+
• Documentation builder
|
| 11 |
+
• Autonomous Brain planning
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
# =========================================================
|
| 15 |
+
# ANALYSIS TASKS
|
| 16 |
+
# =========================================================
|
| 17 |
+
|
| 18 |
+
from publisher.tasks.transcribe import run as transcribe
|
| 19 |
+
from publisher.tasks.subtitles import run as subtitles
|
| 20 |
+
from publisher.tasks.highlights import run as highlights
|
| 21 |
+
from publisher.tasks.viral_score import run as viral_score
|
| 22 |
+
from publisher.tasks.strategy import run as strategy
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
# =========================================================
|
| 26 |
+
# RENDER TASKS
|
| 27 |
+
# =========================================================
|
| 28 |
+
|
| 29 |
+
from publisher.tasks.render import run as render
|
| 30 |
+
from publisher.tasks.generate_thumbnail import run as generate_thumbnail
|
| 31 |
+
from publisher.tasks.generate_metadata import run as generate_metadata
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# =========================================================
|
| 35 |
+
# PUBLISH TASKS
|
| 36 |
+
# =========================================================
|
| 37 |
+
|
| 38 |
+
from publisher.tasks.publish_tiktok import run as publish_tiktok
|
| 39 |
+
from publisher.tasks.publish_youtube import run as publish_youtube
|
| 40 |
+
from publisher.tasks.publish_instagram import run as publish_instagram
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# =========================================================
|
| 44 |
+
# SYSTEM TASKS
|
| 45 |
+
# =========================================================
|
| 46 |
+
|
| 47 |
+
from publisher.tasks.batch_runner import run as batch_runner
|
| 48 |
+
from publisher.tasks.autonomous_mode import run as autonomous_mode
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# =========================================================
|
| 52 |
+
# TASK REGISTRY
|
| 53 |
+
# =========================================================
|
| 54 |
+
|
| 55 |
+
TASKS = {
|
| 56 |
+
|
| 57 |
+
# -----------------------------------------------------
|
| 58 |
+
# ANALYSIS
|
| 59 |
+
# -----------------------------------------------------
|
| 60 |
+
"analysis": [
|
| 61 |
+
|
| 62 |
+
{
|
| 63 |
+
"id": "transcribe",
|
| 64 |
+
"name": "Transcribe",
|
| 65 |
+
"description": "Generate transcript from video/audio",
|
| 66 |
+
"inputs": ["file", "url_input"],
|
| 67 |
+
"output": "json",
|
| 68 |
+
"handler": transcribe,
|
| 69 |
+
},
|
| 70 |
+
|
| 71 |
+
{
|
| 72 |
+
"id": "subtitles",
|
| 73 |
+
"name": "Subtitles",
|
| 74 |
+
"description": "Generate SRT subtitles",
|
| 75 |
+
"inputs": ["file", "url_input"],
|
| 76 |
+
"output": "file",
|
| 77 |
+
"handler": subtitles,
|
| 78 |
+
},
|
| 79 |
+
|
| 80 |
+
{
|
| 81 |
+
"id": "highlights",
|
| 82 |
+
"name": "Highlights",
|
| 83 |
+
"description": "Detect viral highlight segments",
|
| 84 |
+
"inputs": ["file"],
|
| 85 |
+
"output": "json",
|
| 86 |
+
"handler": highlights,
|
| 87 |
+
},
|
| 88 |
+
|
| 89 |
+
{
|
| 90 |
+
"id": "viral-score",
|
| 91 |
+
"name": "Viral Score",
|
| 92 |
+
"description": "AI virality prediction",
|
| 93 |
+
"inputs": ["file"],
|
| 94 |
+
"output": "json",
|
| 95 |
+
"handler": viral_score,
|
| 96 |
+
},
|
| 97 |
+
|
| 98 |
+
{
|
| 99 |
+
"id": "strategy",
|
| 100 |
+
"name": "Strategy",
|
| 101 |
+
"description": "Content strategy generation",
|
| 102 |
+
"inputs": ["file"],
|
| 103 |
+
"output": "json",
|
| 104 |
+
"handler": strategy,
|
| 105 |
+
},
|
| 106 |
+
],
|
| 107 |
+
|
| 108 |
+
# -----------------------------------------------------
|
| 109 |
+
# RENDER
|
| 110 |
+
# -----------------------------------------------------
|
| 111 |
+
"render": [
|
| 112 |
+
|
| 113 |
+
{
|
| 114 |
+
"id": "render",
|
| 115 |
+
"name": "Render Video",
|
| 116 |
+
"description": "Render final short-form video",
|
| 117 |
+
"inputs": ["file", "url_input"],
|
| 118 |
+
"output": "video",
|
| 119 |
+
"video_output": True,
|
| 120 |
+
"handler": render,
|
| 121 |
+
},
|
| 122 |
+
|
| 123 |
+
{
|
| 124 |
+
"id": "generate-thumbnail",
|
| 125 |
+
"name": "Thumbnail",
|
| 126 |
+
"description": "Generate AI thumbnail",
|
| 127 |
+
"inputs": ["file"],
|
| 128 |
+
"output": "image",
|
| 129 |
+
"handler": generate_thumbnail,
|
| 130 |
+
},
|
| 131 |
+
|
| 132 |
+
{
|
| 133 |
+
"id": "generate-metadata",
|
| 134 |
+
"name": "Metadata",
|
| 135 |
+
"description": "Generate captions, hashtags, titles",
|
| 136 |
+
"inputs": ["file"],
|
| 137 |
+
"output": "json",
|
| 138 |
+
"handler": generate_metadata,
|
| 139 |
+
},
|
| 140 |
+
],
|
| 141 |
+
|
| 142 |
+
# -----------------------------------------------------
|
| 143 |
+
# PUBLISH
|
| 144 |
+
# -----------------------------------------------------
|
| 145 |
+
"publish": [
|
| 146 |
+
|
| 147 |
+
{
|
| 148 |
+
"id": "publish-tiktok",
|
| 149 |
+
"name": "Publish TikTok",
|
| 150 |
+
"description": "Upload video to TikTok",
|
| 151 |
+
"inputs": ["file"],
|
| 152 |
+
"output": "json",
|
| 153 |
+
"handler": publish_tiktok,
|
| 154 |
+
},
|
| 155 |
+
|
| 156 |
+
{
|
| 157 |
+
"id": "publish-youtube",
|
| 158 |
+
"name": "Publish YouTube",
|
| 159 |
+
"description": "Upload YouTube Short",
|
| 160 |
+
"inputs": ["file"],
|
| 161 |
+
"output": "json",
|
| 162 |
+
"handler": publish_youtube,
|
| 163 |
+
},
|
| 164 |
+
|
| 165 |
+
{
|
| 166 |
+
"id": "publish-instagram",
|
| 167 |
+
"name": "Publish Instagram",
|
| 168 |
+
"description": "Upload Instagram Reel",
|
| 169 |
+
"inputs": ["file"],
|
| 170 |
+
"output": "json",
|
| 171 |
+
"handler": publish_instagram,
|
| 172 |
+
},
|
| 173 |
+
],
|
| 174 |
+
|
| 175 |
+
# -----------------------------------------------------
|
| 176 |
+
# SYSTEM
|
| 177 |
+
# -----------------------------------------------------
|
| 178 |
+
"system": [
|
| 179 |
+
|
| 180 |
+
{
|
| 181 |
+
"id": "batch",
|
| 182 |
+
"name": "Batch Processor",
|
| 183 |
+
"description": "Execute batch pipeline",
|
| 184 |
+
"inputs": [],
|
| 185 |
+
"output": "json",
|
| 186 |
+
"handler": batch_runner,
|
| 187 |
+
},
|
| 188 |
+
|
| 189 |
+
{
|
| 190 |
+
"id": "autonomous",
|
| 191 |
+
"name": "Autonomous Mode",
|
| 192 |
+
"description": "Start autonomous AI publisher",
|
| 193 |
+
"inputs": [],
|
| 194 |
+
"output": "json",
|
| 195 |
+
"handler": autonomous_mode,
|
| 196 |
+
},
|
| 197 |
+
],
|
| 198 |
+
}
|
fonts/TikTok-Bold.ttf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e7b158d73db62c81c219602a5aef762aa0f40defa31d0e5dc72a2221b6d59fa
|
| 3 |
+
size 131
|
ingestion/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Initialize package
|
ingestion/base64_loader.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import tempfile
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def decode_base64(data):
|
| 6 |
+
|
| 7 |
+
header, encoded = data.split(",", 1)
|
| 8 |
+
|
| 9 |
+
binary = base64.b64decode(encoded)
|
| 10 |
+
|
| 11 |
+
tmp = tempfile.NamedTemporaryFile(
|
| 12 |
+
delete=False,
|
| 13 |
+
suffix=".mp4"
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
tmp.write(binary)
|
| 17 |
+
tmp.close()
|
| 18 |
+
|
| 19 |
+
return tmp.name
|
ingestion/classifiers.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from urllib.parse import urlparse
|
| 3 |
+
|
| 4 |
+
def classify_source(source: str):
|
| 5 |
+
|
| 6 |
+
if not source:
|
| 7 |
+
raise Exception("Empty source")
|
| 8 |
+
|
| 9 |
+
if os.path.exists(source):
|
| 10 |
+
return "local"
|
| 11 |
+
|
| 12 |
+
if source.endswith(".mp4"):
|
| 13 |
+
return "direct"
|
| 14 |
+
|
| 15 |
+
domain = urlparse(source).netloc.lower()
|
| 16 |
+
|
| 17 |
+
if "youtube" in domain or "youtu.be" in domain:
|
| 18 |
+
return "youtube"
|
| 19 |
+
|
| 20 |
+
if "tiktok" in domain:
|
| 21 |
+
return "tiktok"
|
| 22 |
+
|
| 23 |
+
if "instagram" in domain:
|
| 24 |
+
return "instagram"
|
| 25 |
+
|
| 26 |
+
if "facebook" in domain:
|
| 27 |
+
return "facebook"
|
| 28 |
+
|
| 29 |
+
return "unknown"
|
ingestion/cookies/youtube.txt
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"domain": ".youtube.com",
|
| 4 |
+
"expirationDate": 1802677401.949858,
|
| 5 |
+
"hostOnly": false,
|
| 6 |
+
"httpOnly": true,
|
| 7 |
+
"name": "__Secure-3PSID",
|
| 8 |
+
"path": "/",
|
| 9 |
+
"sameSite": "no_restriction",
|
| 10 |
+
"secure": true,
|
| 11 |
+
"session": false,
|
| 12 |
+
"storeId": null,
|
| 13 |
+
"value": "g.a0005ggHIKZHTs9zoAq0BBIjMwWabLARvs6GknU_-0exEvU5DGUwqAzR62E1h_wa5LLS12PspgACgYKAdASARYSFQHGX2MijjE-c2eX18fNOAWSCVLwmBoVAUF8yKq04Fen2HnSxnxT5K2Fpd4j0076"
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"domain": ".youtube.com",
|
| 17 |
+
"expirationDate": 1809905181.409847,
|
| 18 |
+
"hostOnly": false,
|
| 19 |
+
"httpOnly": false,
|
| 20 |
+
"name": "SIDCC",
|
| 21 |
+
"path": "/",
|
| 22 |
+
"sameSite": null,
|
| 23 |
+
"secure": false,
|
| 24 |
+
"session": false,
|
| 25 |
+
"storeId": null,
|
| 26 |
+
"value": "AKEyXzUpR9UA-g9qoHyFYOWJ1CGoWZ5tgVDYm4bEHntfZ5iN6z6RVL8K33zjjp4slMle6hPdIw"
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"domain": ".youtube.com",
|
| 30 |
+
"expirationDate": 1802677401.948968,
|
| 31 |
+
"hostOnly": false,
|
| 32 |
+
"httpOnly": false,
|
| 33 |
+
"name": "SID",
|
| 34 |
+
"path": "/",
|
| 35 |
+
"sameSite": null,
|
| 36 |
+
"secure": false,
|
| 37 |
+
"session": false,
|
| 38 |
+
"storeId": null,
|
| 39 |
+
"value": "g.a0005ggHIKZHTs9zoAq0BBIjMwWabLARvs6GknU_-0exEvU5DGUwA_FG-6Wg9ggcFsmCVhmjswACgYKAcMSARYSFQHGX2Mi4I_6gfKzdtxZN4ZX_kshjxoVAUF8yKqpv1_RfX_zFLUrl3TCcBZg0076"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"domain": ".youtube.com",
|
| 43 |
+
"expirationDate": 1809905178.393538,
|
| 44 |
+
"hostOnly": false,
|
| 45 |
+
"httpOnly": true,
|
| 46 |
+
"name": "__Secure-1PSIDTS",
|
| 47 |
+
"path": "/",
|
| 48 |
+
"sameSite": null,
|
| 49 |
+
"secure": true,
|
| 50 |
+
"session": false,
|
| 51 |
+
"storeId": null,
|
| 52 |
+
"value": "sidts-CjUBhkeRd5PWbJP13KNRyvfxmZVpHrCLWiB90qS5dK1VtBZHEqYG_idUO7UYEmnajmrCqhVkWxAA"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"domain": ".youtube.com",
|
| 56 |
+
"expirationDate": 1802677401.947697,
|
| 57 |
+
"hostOnly": false,
|
| 58 |
+
"httpOnly": false,
|
| 59 |
+
"name": "SAPISID",
|
| 60 |
+
"path": "/",
|
| 61 |
+
"sameSite": null,
|
| 62 |
+
"secure": true,
|
| 63 |
+
"session": false,
|
| 64 |
+
"storeId": null,
|
| 65 |
+
"value": "KqRpYGBu4XUSI9Dm/AZQgHuYmw_rieK8yf"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"domain": ".youtube.com",
|
| 69 |
+
"expirationDate": 1809905181.410137,
|
| 70 |
+
"hostOnly": false,
|
| 71 |
+
"httpOnly": true,
|
| 72 |
+
"name": "__Secure-1PSIDCC",
|
| 73 |
+
"path": "/",
|
| 74 |
+
"sameSite": null,
|
| 75 |
+
"secure": true,
|
| 76 |
+
"session": false,
|
| 77 |
+
"storeId": null,
|
| 78 |
+
"value": "AKEyXzWZL0sioZ0UszpeTFyWSxRW5lbbqZGrHkU8PxHlV-jCSWbkudpG9_cQRuSIw4qB_qoxPw"
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"domain": ".youtube.com",
|
| 82 |
+
"expirationDate": 1802677401.947169,
|
| 83 |
+
"hostOnly": false,
|
| 84 |
+
"httpOnly": true,
|
| 85 |
+
"name": "SSID",
|
| 86 |
+
"path": "/",
|
| 87 |
+
"sameSite": null,
|
| 88 |
+
"secure": true,
|
| 89 |
+
"session": false,
|
| 90 |
+
"storeId": null,
|
| 91 |
+
"value": "A4IKjzFxcSCTwJsHj"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"domain": ".youtube.com",
|
| 95 |
+
"expirationDate": 1802677401.948117,
|
| 96 |
+
"hostOnly": false,
|
| 97 |
+
"httpOnly": false,
|
| 98 |
+
"name": "__Secure-1PAPISID",
|
| 99 |
+
"path": "/",
|
| 100 |
+
"sameSite": null,
|
| 101 |
+
"secure": true,
|
| 102 |
+
"session": false,
|
| 103 |
+
"storeId": null,
|
| 104 |
+
"value": "KqRpYGBu4XUSI9Dm/AZQgHuYmw_rieK8yf"
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"domain": ".youtube.com",
|
| 108 |
+
"expirationDate": 1802677401.949424,
|
| 109 |
+
"hostOnly": false,
|
| 110 |
+
"httpOnly": true,
|
| 111 |
+
"name": "__Secure-1PSID",
|
| 112 |
+
"path": "/",
|
| 113 |
+
"sameSite": null,
|
| 114 |
+
"secure": true,
|
| 115 |
+
"session": false,
|
| 116 |
+
"storeId": null,
|
| 117 |
+
"value": "g.a0005ggHIKZHTs9zoAq0BBIjMwWabLARvs6GknU_-0exEvU5DGUwDX3_lEm4gf-uTFeESekcrQACgYKAd0SARYSFQHGX2MiwxRcpnYfY1bFNKTsueE3oxoVAUF8yKr-LKssrIr-F9tOdzCc1xo70076"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"domain": ".youtube.com",
|
| 121 |
+
"expirationDate": 1802677401.948542,
|
| 122 |
+
"hostOnly": false,
|
| 123 |
+
"httpOnly": false,
|
| 124 |
+
"name": "__Secure-3PAPISID",
|
| 125 |
+
"path": "/",
|
| 126 |
+
"sameSite": "no_restriction",
|
| 127 |
+
"secure": true,
|
| 128 |
+
"session": false,
|
| 129 |
+
"storeId": null,
|
| 130 |
+
"value": "KqRpYGBu4XUSI9Dm/AZQgHuYmw_rieK8yf"
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"domain": ".youtube.com",
|
| 134 |
+
"expirationDate": 1809905181.410357,
|
| 135 |
+
"hostOnly": false,
|
| 136 |
+
"httpOnly": true,
|
| 137 |
+
"name": "__Secure-3PSIDCC",
|
| 138 |
+
"path": "/",
|
| 139 |
+
"sameSite": "no_restriction",
|
| 140 |
+
"secure": true,
|
| 141 |
+
"session": false,
|
| 142 |
+
"storeId": null,
|
| 143 |
+
"value": "AKEyXzWCPVCW7k3dwkKSsjV05nT9K7hC-wPUt49b_JuXW45TFu08ti7lfQsXqgfeDezseQCbZA"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"domain": ".youtube.com",
|
| 147 |
+
"expirationDate": 1809905178.39415,
|
| 148 |
+
"hostOnly": false,
|
| 149 |
+
"httpOnly": true,
|
| 150 |
+
"name": "__Secure-3PSIDTS",
|
| 151 |
+
"path": "/",
|
| 152 |
+
"sameSite": "no_restriction",
|
| 153 |
+
"secure": true,
|
| 154 |
+
"session": false,
|
| 155 |
+
"storeId": null,
|
| 156 |
+
"value": "sidts-CjUBhkeRd5PWbJP13KNRyvfxmZVpHrCLWiB90qS5dK1VtBZHEqYG_idUO7UYEmnajmrCqhVkWxAA"
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"domain": ".youtube.com",
|
| 160 |
+
"expirationDate": 1802677401.947412,
|
| 161 |
+
"hostOnly": false,
|
| 162 |
+
"httpOnly": false,
|
| 163 |
+
"name": "APISID",
|
| 164 |
+
"path": "/",
|
| 165 |
+
"sameSite": null,
|
| 166 |
+
"secure": false,
|
| 167 |
+
"session": false,
|
| 168 |
+
"storeId": null,
|
| 169 |
+
"value": "36o598sqGfxJDffW/AyB1SJrxHZGXJ3xdj"
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"domain": ".youtube.com",
|
| 173 |
+
"expirationDate": 1802677401.946773,
|
| 174 |
+
"hostOnly": false,
|
| 175 |
+
"httpOnly": true,
|
| 176 |
+
"name": "HSID",
|
| 177 |
+
"path": "/",
|
| 178 |
+
"sameSite": null,
|
| 179 |
+
"secure": false,
|
| 180 |
+
"session": false,
|
| 181 |
+
"storeId": null,
|
| 182 |
+
"value": "AhAjQv9yddkNo7Ro3"
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"domain": ".youtube.com",
|
| 186 |
+
"expirationDate": 1812929163.824495,
|
| 187 |
+
"hostOnly": false,
|
| 188 |
+
"httpOnly": true,
|
| 189 |
+
"name": "LOGIN_INFO",
|
| 190 |
+
"path": "/",
|
| 191 |
+
"sameSite": "no_restriction",
|
| 192 |
+
"secure": true,
|
| 193 |
+
"session": false,
|
| 194 |
+
"storeId": null,
|
| 195 |
+
"value": "AFmmF2swRgIhAMcn3H8l0bXlZVgci9p5SvScepya1xAObUTRdJgGH5qqAiEAnQveY5dETRyStRQqDag0lUXHcTbRJkvDQ2Ge8GiDSBk:QUQ3MjNmd3IweGRfR3VIeVRCUGRiVHlkM1RxUXZjM1JnNUZCc0Nvd3lYV1l6UlpCdVVLdHR2cVgwU0UxUFloMkkwVlhXdWtMalQyMHZIVkc5eTFDU3dGSnQ1QWk5Z0N6XzZOTzVuRThBZHF3VkZsVlNpdmZ0TWExTk5jLUVxelhRQVF4M1p3RTRYSnU2dV9LQ0VoTllEcUNhZnBER3o2SW5R"
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"domain": ".youtube.com",
|
| 199 |
+
"expirationDate": 1812929169.133873,
|
| 200 |
+
"hostOnly": false,
|
| 201 |
+
"httpOnly": false,
|
| 202 |
+
"name": "PREF",
|
| 203 |
+
"path": "/",
|
| 204 |
+
"sameSite": null,
|
| 205 |
+
"secure": true,
|
| 206 |
+
"session": false,
|
| 207 |
+
"storeId": null,
|
| 208 |
+
"value": "f6=40000000&tz=Africa.Lagos"
|
| 209 |
+
}
|
| 210 |
+
]
|
ingestion/downloader.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import requests
|
| 2 |
+
import tempfile
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def download_file(url):
|
| 6 |
+
|
| 7 |
+
r = requests.get(url, stream=True, timeout=120)
|
| 8 |
+
|
| 9 |
+
r.raise_for_status()
|
| 10 |
+
|
| 11 |
+
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
|
| 12 |
+
|
| 13 |
+
for chunk in r.iter_content(1024 * 1024):
|
| 14 |
+
tmp.write(chunk)
|
| 15 |
+
|
| 16 |
+
tmp.close()
|
| 17 |
+
|
| 18 |
+
return tmp.name
|
ingestion/normalizer.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import subprocess
|
| 2 |
+
import uuid
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
def normalize_video(path):
|
| 6 |
+
|
| 7 |
+
output = f"jobs/norm_{uuid.uuid4()}.mp4"
|
| 8 |
+
|
| 9 |
+
cmd = [
|
| 10 |
+
"ffmpeg",
|
| 11 |
+
"-y",
|
| 12 |
+
"-i", path,
|
| 13 |
+
"-vf", "scale=1080:-2",
|
| 14 |
+
"-c:v", "libx264",
|
| 15 |
+
"-preset", "veryfast",
|
| 16 |
+
"-crf", "23",
|
| 17 |
+
"-c:a", "aac",
|
| 18 |
+
"-movflags", "+faststart",
|
| 19 |
+
output,
|
| 20 |
+
]
|
| 21 |
+
|
| 22 |
+
subprocess.run(cmd, check=True)
|
| 23 |
+
|
| 24 |
+
return output
|
ingestion/resolver.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import uuid
|
| 3 |
+
import requests
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Optional, Union
|
| 6 |
+
from fastapi import UploadFile
|
| 7 |
+
|
| 8 |
+
# ==============================
|
| 9 |
+
# STORAGE CONFIG
|
| 10 |
+
# ==============================
|
| 11 |
+
|
| 12 |
+
BASE_DIR = Path(__file__).resolve().parents[1]
|
| 13 |
+
UPLOAD_DIR = str(BASE_DIR / "jobs" / "uploads")
|
| 14 |
+
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
| 15 |
+
|
| 16 |
+
CHUNK_SIZE = 1024 * 1024 # 1MB streaming for large media
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# ==============================
|
| 20 |
+
# CORE RESOLVER
|
| 21 |
+
# ==============================
|
| 22 |
+
|
| 23 |
+
def resolve_input(
|
| 24 |
+
source: Optional[str] = None,
|
| 25 |
+
upload: Optional[UploadFile] = None,
|
| 26 |
+
raw_bytes: Optional[bytes] = None
|
| 27 |
+
) -> str:
|
| 28 |
+
"""
|
| 29 |
+
Universal ingestion layer for all pipeline systems.
|
| 30 |
+
|
| 31 |
+
Supports:
|
| 32 |
+
- UploadFile (FastAPI / Gradio)
|
| 33 |
+
- URL download (http/https)
|
| 34 |
+
- Local filesystem path
|
| 35 |
+
- Raw bytes input (future automation nodes)
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
# ----------------------------------------
|
| 39 |
+
# CASE 1: UploadFile (Gradio / FastAPI)
|
| 40 |
+
# ----------------------------------------
|
| 41 |
+
if upload is not None:
|
| 42 |
+
filename = f"{uuid.uuid4()}_{upload.filename or 'upload.mp4'}"
|
| 43 |
+
path = os.path.join(UPLOAD_DIR, filename)
|
| 44 |
+
|
| 45 |
+
with open(path, "wb") as f:
|
| 46 |
+
while True:
|
| 47 |
+
chunk = upload.file.read(CHUNK_SIZE)
|
| 48 |
+
if not chunk:
|
| 49 |
+
break
|
| 50 |
+
f.write(chunk)
|
| 51 |
+
|
| 52 |
+
return path
|
| 53 |
+
|
| 54 |
+
# ----------------------------------------
|
| 55 |
+
# CASE 2: Raw bytes (automation / webhook)
|
| 56 |
+
# ----------------------------------------
|
| 57 |
+
if raw_bytes is not None:
|
| 58 |
+
filename = f"{uuid.uuid4()}.mp4"
|
| 59 |
+
path = os.path.join(UPLOAD_DIR, filename)
|
| 60 |
+
|
| 61 |
+
with open(path, "wb") as f:
|
| 62 |
+
f.write(raw_bytes)
|
| 63 |
+
|
| 64 |
+
return path
|
| 65 |
+
|
| 66 |
+
# ----------------------------------------
|
| 67 |
+
# CASE 3: URL input (YouTube, TikTok, direct mp4)
|
| 68 |
+
# ----------------------------------------
|
| 69 |
+
if source and source.startswith(("http://", "https://")):
|
| 70 |
+
|
| 71 |
+
filename = f"{uuid.uuid4()}.mp4"
|
| 72 |
+
path = os.path.join(UPLOAD_DIR, filename)
|
| 73 |
+
|
| 74 |
+
headers = {
|
| 75 |
+
"User-Agent": "Mozilla/5.0 (compatible; BasyxBot/1.0)"
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
with requests.get(source, stream=True, headers=headers, timeout=60) as r:
|
| 79 |
+
r.raise_for_status()
|
| 80 |
+
|
| 81 |
+
with open(path, "wb") as f:
|
| 82 |
+
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
|
| 83 |
+
if chunk:
|
| 84 |
+
f.write(chunk)
|
| 85 |
+
|
| 86 |
+
return path
|
| 87 |
+
|
| 88 |
+
# ----------------------------------------
|
| 89 |
+
# CASE 4: Local file path
|
| 90 |
+
# ----------------------------------------
|
| 91 |
+
if source and os.path.exists(source):
|
| 92 |
+
return source
|
| 93 |
+
|
| 94 |
+
# ----------------------------------------
|
| 95 |
+
# INVALID INPUT HANDLING
|
| 96 |
+
# ----------------------------------------
|
| 97 |
+
raise ValueError(
|
| 98 |
+
"resolve_input failed: no valid source, upload, or raw_bytes provided"
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# ==============================
|
| 103 |
+
# OPTIONAL HELPERS (V11 READY)
|
| 104 |
+
# ==============================
|
| 105 |
+
|
| 106 |
+
def detect_input_type(source: str) -> str:
|
| 107 |
+
"""
|
| 108 |
+
Lightweight classifier for routing decisions upstream.
|
| 109 |
+
"""
|
| 110 |
+
|
| 111 |
+
if source.startswith(("http://", "https://")):
|
| 112 |
+
return "url"
|
| 113 |
+
|
| 114 |
+
if os.path.exists(source):
|
| 115 |
+
return "file"
|
| 116 |
+
|
| 117 |
+
return "unknown"
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def normalize_source(source: str) -> str:
|
| 121 |
+
"""
|
| 122 |
+
Cleans input strings for downstream consistency.
|
| 123 |
+
"""
|
| 124 |
+
if not source:
|
| 125 |
+
return source
|
| 126 |
+
|
| 127 |
+
return source.strip()
|
ingestion/social.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import tempfile
|
| 2 |
+
import yt_dlp
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def download_social(url):
|
| 6 |
+
|
| 7 |
+
output = tempfile.NamedTemporaryFile(
|
| 8 |
+
delete=False,
|
| 9 |
+
suffix=".mp4"
|
| 10 |
+
).name
|
| 11 |
+
|
| 12 |
+
ydl_opts = {
|
| 13 |
+
"outtmpl": output,
|
| 14 |
+
"format": "mp4",
|
| 15 |
+
"quiet": True
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
| 19 |
+
ydl.download([url])
|
| 20 |
+
|
| 21 |
+
return output
|
models/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .kokoro import Kokoro
|
| 2 |
+
from .tokenizer import Tokenizer
|
models/kokoro.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import numpy as np
|
| 3 |
+
import onnxruntime as ort
|
| 4 |
+
|
| 5 |
+
TOKEN_LIMIT = 510
|
| 6 |
+
SAMPLE_RATE = 24_000
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class Kokoro:
|
| 10 |
+
def __init__(self, model_path: str, style_vector_path: str, tokenizer, lang: str = 'en-us') -> None:
|
| 11 |
+
"""
|
| 12 |
+
Initializes the ONNXInference class.
|
| 13 |
+
|
| 14 |
+
Args:
|
| 15 |
+
model_path (str): Path to the ONNX model file.
|
| 16 |
+
style_vector_path (str): Path to the style vector file.
|
| 17 |
+
lang (str): Language code for the tokenizer.
|
| 18 |
+
"""
|
| 19 |
+
self.sess = ort.InferenceSession(model_path)
|
| 20 |
+
self.style_vector_path = style_vector_path
|
| 21 |
+
self.tokenizer = tokenizer
|
| 22 |
+
self.lang = lang
|
| 23 |
+
|
| 24 |
+
def preprocess(self, text):
|
| 25 |
+
"""
|
| 26 |
+
Converts input text to tokenized numerical IDs and loads the style vector.
|
| 27 |
+
|
| 28 |
+
Args:
|
| 29 |
+
text (str): Input text to preprocess.
|
| 30 |
+
|
| 31 |
+
Returns:
|
| 32 |
+
tuple: Tokenized input and corresponding style vector.
|
| 33 |
+
"""
|
| 34 |
+
# Convert text to phonemes and tokenize
|
| 35 |
+
phonemes = self.tokenizer.phonemize(text, lang=self.lang)
|
| 36 |
+
tokenized_phonemes = self.tokenizer.tokenize(phonemes)
|
| 37 |
+
|
| 38 |
+
if not tokenized_phonemes:
|
| 39 |
+
raise ValueError("No tokens found after tokenization")
|
| 40 |
+
|
| 41 |
+
style_vector = torch.load(self.style_vector_path, weights_only=True)
|
| 42 |
+
|
| 43 |
+
if len(tokenized_phonemes) > TOKEN_LIMIT:
|
| 44 |
+
token_chunks = self.split_into_chunks(tokenized_phonemes)
|
| 45 |
+
|
| 46 |
+
tokens_list = []
|
| 47 |
+
styles_list = []
|
| 48 |
+
|
| 49 |
+
for chunk in token_chunks:
|
| 50 |
+
token_chunk = [[0, *chunk, 0]]
|
| 51 |
+
style_chunk = style_vector[len(chunk)].numpy()
|
| 52 |
+
|
| 53 |
+
tokens_list.append(token_chunk)
|
| 54 |
+
styles_list.append(style_chunk)
|
| 55 |
+
|
| 56 |
+
return tokens_list, styles_list
|
| 57 |
+
|
| 58 |
+
style_vector = style_vector[len(tokenized_phonemes)].numpy()
|
| 59 |
+
tokenized_phonemes = [[0, *tokenized_phonemes, 0]]
|
| 60 |
+
|
| 61 |
+
return tokenized_phonemes, style_vector
|
| 62 |
+
|
| 63 |
+
@staticmethod
|
| 64 |
+
def split_into_chunks(tokens):
|
| 65 |
+
"""
|
| 66 |
+
Splits a list of tokens into chunks of size TOKEN_LIMIT.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
tokens (list): List of tokens to split.
|
| 70 |
+
|
| 71 |
+
Returns:
|
| 72 |
+
list: List of token chunks.
|
| 73 |
+
"""
|
| 74 |
+
tokens_chunks = []
|
| 75 |
+
for i in range(0, len(tokens), TOKEN_LIMIT):
|
| 76 |
+
tokens_chunks.append(tokens[i:i+TOKEN_LIMIT])
|
| 77 |
+
return tokens_chunks
|
| 78 |
+
|
| 79 |
+
def infer(self, tokens, style_vector, speed=1.0):
|
| 80 |
+
"""
|
| 81 |
+
Runs inference using the ONNX model.
|
| 82 |
+
|
| 83 |
+
Args:
|
| 84 |
+
tokens (list): Tokenized input for the model.
|
| 85 |
+
style_vector (numpy.ndarray): Style vector for the model.
|
| 86 |
+
speed (float): Speed parameter for inference.
|
| 87 |
+
|
| 88 |
+
Returns:
|
| 89 |
+
numpy.ndarray: Generated audio data.
|
| 90 |
+
"""
|
| 91 |
+
# Perform inference
|
| 92 |
+
audio = self.sess.run(
|
| 93 |
+
None,
|
| 94 |
+
{
|
| 95 |
+
'tokens': tokens,
|
| 96 |
+
'style': style_vector,
|
| 97 |
+
'speed': np.array([speed], dtype=np.float32),
|
| 98 |
+
}
|
| 99 |
+
)[0]
|
| 100 |
+
return audio
|
| 101 |
+
|
| 102 |
+
def generate_audio(self, text, speed=1.0):
|
| 103 |
+
"""
|
| 104 |
+
Full pipeline: preprocess, infer, and save the generated audio.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
text (str): Input text to generate audio from.
|
| 108 |
+
speed (float): Speed parameter for inference.
|
| 109 |
+
"""
|
| 110 |
+
# Preprocess text
|
| 111 |
+
tokenized_data, styles_data = self.preprocess(text)
|
| 112 |
+
|
| 113 |
+
audio_segments = []
|
| 114 |
+
if len(tokenized_data) > 1: # list of token chunks
|
| 115 |
+
for token_chunk, style_chunk in zip(tokenized_data, styles_data):
|
| 116 |
+
audio = self.infer(token_chunk, style_chunk, speed=speed)
|
| 117 |
+
audio_segments.append(audio)
|
| 118 |
+
else: # single token less than input limit
|
| 119 |
+
# Run inference
|
| 120 |
+
audio = self.infer(tokenized_data, styles_data, speed=speed)
|
| 121 |
+
audio_segments.append(audio)
|
| 122 |
+
|
| 123 |
+
full_audio = np.concatenate(audio_segments)
|
| 124 |
+
|
| 125 |
+
return full_audio, SAMPLE_RATE
|
models/tokenizer.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
from phonemizer import backend
|
| 3 |
+
from typing import List
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class Tokenizer:
|
| 7 |
+
def __init__(self):
|
| 8 |
+
self.VOCAB = self._get_vocab()
|
| 9 |
+
self.phonemizers = {
|
| 10 |
+
'en-us': backend.EspeakBackend(language='en-us', preserve_punctuation=True, with_stress=True),
|
| 11 |
+
'en-gb': backend.EspeakBackend(language='en-gb', preserve_punctuation=True, with_stress=True),
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
@staticmethod
|
| 15 |
+
def _get_vocab():
|
| 16 |
+
"""
|
| 17 |
+
Generates a mapping of symbols to integer indices for tokenization.
|
| 18 |
+
|
| 19 |
+
Returns:
|
| 20 |
+
dict: A dictionary where keys are symbols and values are unique integer indices.
|
| 21 |
+
"""
|
| 22 |
+
# Define the symbols
|
| 23 |
+
_pad = "$"
|
| 24 |
+
_punctuation = ';:,.!?¡¿—…"«»“” '
|
| 25 |
+
_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
|
| 26 |
+
_letters_ipa = (
|
| 27 |
+
"ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ"
|
| 28 |
+
)
|
| 29 |
+
symbols = [_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa)
|
| 30 |
+
|
| 31 |
+
# Create a dictionary mapping each symbol to its index
|
| 32 |
+
return {symbol: index for index, symbol in enumerate(symbols)}
|
| 33 |
+
|
| 34 |
+
@staticmethod
|
| 35 |
+
def split_num(num: re.Match) -> str:
|
| 36 |
+
"""
|
| 37 |
+
Processes numeric strings, formatting them as time, years, or other representations.
|
| 38 |
+
|
| 39 |
+
Args:
|
| 40 |
+
num (re.Match): A regex match object representing the numeric string.
|
| 41 |
+
|
| 42 |
+
Returns:
|
| 43 |
+
str: A formatted string based on the numeric input.
|
| 44 |
+
"""
|
| 45 |
+
num = num.group()
|
| 46 |
+
|
| 47 |
+
# Handle time (e.g., "12:30")
|
| 48 |
+
if ':' in num:
|
| 49 |
+
hours, minutes = map(int, num.split(':'))
|
| 50 |
+
if minutes == 0:
|
| 51 |
+
return f"{hours} o'clock"
|
| 52 |
+
elif minutes < 10:
|
| 53 |
+
return f'{hours} oh {minutes}'
|
| 54 |
+
return f'{hours} {minutes}'
|
| 55 |
+
|
| 56 |
+
# Handle years or general numeric cases
|
| 57 |
+
year = int(num[:4])
|
| 58 |
+
if year < 1100 or year % 1000 < 10:
|
| 59 |
+
return num
|
| 60 |
+
|
| 61 |
+
left, right = num[:2], int(num[2:4])
|
| 62 |
+
suffix = 's' if num.endswith('s') else ''
|
| 63 |
+
|
| 64 |
+
# Format years
|
| 65 |
+
if 100 <= year % 1000 <= 999:
|
| 66 |
+
if right == 0:
|
| 67 |
+
return f'{left} hundred{suffix}'
|
| 68 |
+
elif right < 10:
|
| 69 |
+
return f'{left} oh {right}{suffix}'
|
| 70 |
+
return f'{left} {right}{suffix}'
|
| 71 |
+
|
| 72 |
+
@staticmethod
|
| 73 |
+
def flip_money(match: re.Match) -> str:
|
| 74 |
+
"""
|
| 75 |
+
Converts monetary values to a textual representation.
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
m (re.Match): A regex match object representing the monetary value.
|
| 79 |
+
|
| 80 |
+
Returns:
|
| 81 |
+
str: A formatted string describing the monetary value.
|
| 82 |
+
"""
|
| 83 |
+
m = m.group()
|
| 84 |
+
currency = 'dollar' if m[0] == '$' else 'pound'
|
| 85 |
+
|
| 86 |
+
# Handle whole amounts (e.g., "$10", "£20")
|
| 87 |
+
if '.' not in m:
|
| 88 |
+
singular = '' if m[1:] == '1' else 's'
|
| 89 |
+
return f'{m[1:]} {currency}{singular}'
|
| 90 |
+
|
| 91 |
+
# Handle amounts with decimals (e.g., "$10.50", "£5.25")
|
| 92 |
+
whole, cents = m[1:].split('.')
|
| 93 |
+
singular = '' if whole == '1' else 's'
|
| 94 |
+
cents = int(cents.ljust(2, '0')) # Ensure 2 decimal places
|
| 95 |
+
coins = f"cent{'' if cents == 1 else 's'}" if m[0] == '$' else ('penny' if cents == 1 else 'pence')
|
| 96 |
+
return f'{whole} {currency}{singular} and {cents} {coins}'
|
| 97 |
+
|
| 98 |
+
@staticmethod
|
| 99 |
+
def point_num(match):
|
| 100 |
+
whole, fractional = match.group().split('.')
|
| 101 |
+
return ' point '.join([whole, ' '.join(fractional)])
|
| 102 |
+
|
| 103 |
+
def normalize_text(self, text: str) -> str:
|
| 104 |
+
"""
|
| 105 |
+
Normalizes input text by replacing special characters, punctuation, and applying custom transformations.
|
| 106 |
+
|
| 107 |
+
Args:
|
| 108 |
+
text (str): Input text to normalize.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
str: Normalized text.
|
| 112 |
+
"""
|
| 113 |
+
# Replace specific characters with standardized versions
|
| 114 |
+
replacements = {
|
| 115 |
+
chr(8216): "'", # Left single quotation mark
|
| 116 |
+
chr(8217): "'", # Right single quotation mark
|
| 117 |
+
'«': chr(8220), # Left double angle quotation mark to left double quotation mark
|
| 118 |
+
'»': chr(8221), # Right double angle quotation mark to right double quotation mark
|
| 119 |
+
chr(8220): '"', # Left double quotation mark
|
| 120 |
+
chr(8221): '"', # Right double quotation mark
|
| 121 |
+
'(': '«', # Replace parentheses with angle quotation marks
|
| 122 |
+
')': '»'
|
| 123 |
+
}
|
| 124 |
+
for old, new in replacements.items():
|
| 125 |
+
text = text.replace(old, new)
|
| 126 |
+
|
| 127 |
+
# Replace punctuation and add spaces
|
| 128 |
+
punctuation_replacements = {
|
| 129 |
+
'、': ',',
|
| 130 |
+
'。': '.',
|
| 131 |
+
'!': '!',
|
| 132 |
+
',': ',',
|
| 133 |
+
':': ':',
|
| 134 |
+
';': ';',
|
| 135 |
+
'?': '?',
|
| 136 |
+
}
|
| 137 |
+
for old, new in punctuation_replacements.items():
|
| 138 |
+
text = text.replace(old, new + ' ')
|
| 139 |
+
|
| 140 |
+
# Apply regex-based replacements
|
| 141 |
+
text = re.sub(r'[^\S\n]', ' ', text)
|
| 142 |
+
text = re.sub(r' +', ' ', text)
|
| 143 |
+
text = re.sub(r'(?<=\n) +(?=\n)', '', text)
|
| 144 |
+
|
| 145 |
+
# Expand abbreviations and handle special cases
|
| 146 |
+
abbreviation_patterns = [
|
| 147 |
+
(r'\bD[Rr]\.(?= [A-Z])', 'Doctor'),
|
| 148 |
+
(r'\b(?:Mr\.|MR\.(?= [A-Z]))', 'Mister'),
|
| 149 |
+
(r'\b(?:Ms\.|MS\.(?= [A-Z]))', 'Miss'),
|
| 150 |
+
(r'\b(?:Mrs\.|MRS\.(?= [A-Z]))', 'Mrs'),
|
| 151 |
+
(r'\betc\.(?! [A-Z])', 'etc'),
|
| 152 |
+
(r'(?i)\b(y)eah?\b', r"\1e'a"),
|
| 153 |
+
]
|
| 154 |
+
for pattern, replacement in abbreviation_patterns:
|
| 155 |
+
text = re.sub(pattern, replacement, text)
|
| 156 |
+
|
| 157 |
+
# Handle numbers and monetary values
|
| 158 |
+
text = re.sub(r'\d*\.\d+|\b\d{4}s?\b|(?<!:)\b(?:[1-9]|1[0-2]):[0-5]\d\b(?!:)', self.split_num, text)
|
| 159 |
+
text = re.sub(r'(?<=\d),(?=\d)', '', text) # Remove commas from numbers
|
| 160 |
+
text = re.sub(
|
| 161 |
+
r'(?i)[$£]\d+(?:\.\d+)?(?: hundred| thousand| (?:[bm]|tr)illion)*\b|[$£]\d+\.\d\d?\b',
|
| 162 |
+
self.flip_money,
|
| 163 |
+
text
|
| 164 |
+
)
|
| 165 |
+
text = re.sub(r'\d*\.\d+', self.point_num, text)
|
| 166 |
+
text = re.sub(r'(?<=\d)-(?=\d)', ' to ', text)
|
| 167 |
+
|
| 168 |
+
# Handle possessives and specific letter cases
|
| 169 |
+
text = re.sub(r'(?<=\d)S', ' S', text)
|
| 170 |
+
text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text)
|
| 171 |
+
text = re.sub(r"(?<=X')S\b", 's', text)
|
| 172 |
+
|
| 173 |
+
# Handle abbreviations with dots
|
| 174 |
+
text = re.sub(r'(?:[A-Za-z]\.){2,} [a-z]', lambda m: m.group().replace('.', '-'), text)
|
| 175 |
+
text = re.sub(r'(?i)(?<=[A-Z])\.(?=[A-Z])', '-', text)
|
| 176 |
+
|
| 177 |
+
return text.strip()
|
| 178 |
+
|
| 179 |
+
def tokenize(self, phonemes: str) -> List[int]:
|
| 180 |
+
"""
|
| 181 |
+
Tokenizes a given string into a list of indices based on VOCAB.
|
| 182 |
+
|
| 183 |
+
Args:
|
| 184 |
+
text (str): Input string to tokenize.
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
list: A list of integer indices corresponding to the characters in the input string.
|
| 188 |
+
"""
|
| 189 |
+
return [self.VOCAB[x] for x in phonemes if x in self.VOCAB]
|
| 190 |
+
|
| 191 |
+
def phonemize(self, text: str, lang: str = 'en-us', normalize: bool = True) -> str:
|
| 192 |
+
"""
|
| 193 |
+
Converts text to phonemes using the specified language phonemizer and applies normalization.
|
| 194 |
+
|
| 195 |
+
Args:
|
| 196 |
+
text (str): Input text to be phonemized.
|
| 197 |
+
lang (str): Language identifier ('en-us' or 'en-gb') for selecting the phonemizer.
|
| 198 |
+
normalize (bool): Whether to normalize the text before phonemization.
|
| 199 |
+
|
| 200 |
+
Returns:
|
| 201 |
+
str: A processed string of phonemes.
|
| 202 |
+
"""
|
| 203 |
+
# Normalize text if required
|
| 204 |
+
if normalize:
|
| 205 |
+
text = self.normalize_text(text)
|
| 206 |
+
|
| 207 |
+
# Generate phonemes using the specified phonemizer
|
| 208 |
+
if lang not in self.phonemizers:
|
| 209 |
+
print(f"Language '{lang}' not supported. Defaulting to 'en-us'.")
|
| 210 |
+
lang = 'en-us'
|
| 211 |
+
|
| 212 |
+
phonemes = self.phonemizers[lang].phonemize([text])
|
| 213 |
+
phonemes = phonemes[0] if phonemes else ''
|
| 214 |
+
|
| 215 |
+
# Apply custom phoneme replacements
|
| 216 |
+
replacements = {
|
| 217 |
+
'kəkˈoːɹoʊ': 'kˈoʊkəɹoʊ',
|
| 218 |
+
'kəkˈɔːɹəʊ': 'kˈəʊkəɹəʊ',
|
| 219 |
+
'ʲ': 'j',
|
| 220 |
+
'r': 'ɹ',
|
| 221 |
+
'x': 'k',
|
| 222 |
+
'ɬ': 'l',
|
| 223 |
+
}
|
| 224 |
+
for old, new in replacements.items():
|
| 225 |
+
phonemes = phonemes.replace(old, new)
|
| 226 |
+
|
| 227 |
+
# Apply regex-based replacements
|
| 228 |
+
phonemes = re.sub(r'(?<=[a-zɹː])(?=hˈʌndɹɪd)', ' ', phonemes)
|
| 229 |
+
phonemes = re.sub(r' z(?=[;:,.!?¡¿—…"«»“” ]|$)', 'z', phonemes)
|
| 230 |
+
|
| 231 |
+
# Additional language-specific rules
|
| 232 |
+
if lang == 'a':
|
| 233 |
+
phonemes = re.sub(r'(?<=nˈaɪn)ti(?!ː)', 'di', phonemes)
|
| 234 |
+
|
| 235 |
+
# Filter out characters not in VOCAB
|
| 236 |
+
phonemes = ''.join(filter(lambda p: p in self.VOCAB, phonemes))
|
| 237 |
+
|
| 238 |
+
return phonemes.strip()
|
publisher/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Publisher package
|
publisher/account_manager.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/account_manager.py
|
| 2 |
+
|
| 3 |
+
from publisher.oauth.storage import load_tokens
|
| 4 |
+
|
| 5 |
+
SUPPORTED = [
|
| 6 |
+
"youtube",
|
| 7 |
+
"tiktok",
|
| 8 |
+
"meta"
|
| 9 |
+
]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def connected_accounts(user_id):
|
| 13 |
+
|
| 14 |
+
accounts = []
|
| 15 |
+
|
| 16 |
+
for p in SUPPORTED:
|
| 17 |
+
token = load_tokens(user_id, p)
|
| 18 |
+
if token:
|
| 19 |
+
accounts.append({
|
| 20 |
+
"platform": p,
|
| 21 |
+
"token": token
|
| 22 |
+
})
|
| 23 |
+
|
| 24 |
+
return accounts
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def choose_platforms(strategy, accounts):
|
| 28 |
+
|
| 29 |
+
if strategy == "all":
|
| 30 |
+
return accounts
|
| 31 |
+
|
| 32 |
+
if strategy == "short_video":
|
| 33 |
+
return [
|
| 34 |
+
a for a in accounts
|
| 35 |
+
if a["platform"] in ["youtube", "tiktok", "meta"]
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
return accounts[:1]
|
publisher/ai/gemini_client.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import time
|
| 3 |
+
import random
|
| 4 |
+
import logging
|
| 5 |
+
|
| 6 |
+
logger = logging.getLogger("gemini-client")
|
| 7 |
+
|
| 8 |
+
# =========================
|
| 9 |
+
# MODEL POOL (Gemini 3 Era)
|
| 10 |
+
# =========================
|
| 11 |
+
|
| 12 |
+
PRIMARY_MODELS = [
|
| 13 |
+
"gemini-3.1-pro",
|
| 14 |
+
"gemini-3.1-flash",
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
FALLBACK_MODELS = [
|
| 18 |
+
"gemini-2.5-flash",
|
| 19 |
+
"gemini-3.1-flash-lite",
|
| 20 |
+
]
|
| 21 |
+
|
| 22 |
+
# =========================
|
| 23 |
+
# SIMPLE QUOTA TRACKER
|
| 24 |
+
# =========================
|
| 25 |
+
_model_fail_count = {
|
| 26 |
+
"gemini-3.1-pro": 0,
|
| 27 |
+
"gemini-3.1-flash": 0,
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
MAX_FAILS = 3
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# =========================
|
| 35 |
+
# CORE MODEL RESOLVER
|
| 36 |
+
# =========================
|
| 37 |
+
|
| 38 |
+
def _pick_model():
|
| 39 |
+
"""
|
| 40 |
+
Select best available model with fallback logic.
|
| 41 |
+
"""
|
| 42 |
+
for m in PRIMARY_MODELS:
|
| 43 |
+
if _model_fail_count.get(m, 0) < MAX_FAILS:
|
| 44 |
+
return m
|
| 45 |
+
|
| 46 |
+
return random.choice(FALLBACK_MODELS)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# =========================
|
| 50 |
+
# MAIN CLIENT INTERFACE
|
| 51 |
+
# =========================
|
| 52 |
+
|
| 53 |
+
def get_model():
|
| 54 |
+
"""
|
| 55 |
+
Public entrypoint used by publisher_ai.
|
| 56 |
+
Returns active Gemini model name.
|
| 57 |
+
"""
|
| 58 |
+
model = _pick_model()
|
| 59 |
+
logger.info(f"[Gemini] Selected model: {model}")
|
| 60 |
+
return model
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def safe_generate(prompt: str, client_callable):
|
| 64 |
+
"""
|
| 65 |
+
Wrapper for Gemini calls with automatic fallback.
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
last_error = None
|
| 69 |
+
|
| 70 |
+
for _ in range(3):
|
| 71 |
+
model = _pick_model()
|
| 72 |
+
|
| 73 |
+
try:
|
| 74 |
+
result = client_callable(model, prompt)
|
| 75 |
+
return result
|
| 76 |
+
|
| 77 |
+
except Exception as e:
|
| 78 |
+
last_error = e
|
| 79 |
+
_model_fail_count[model] = _model_fail_count.get(model, 0) + 1
|
| 80 |
+
logger.warning(f"[Gemini FAIL] {model}: {str(e)}")
|
| 81 |
+
time.sleep(0.5)
|
| 82 |
+
|
| 83 |
+
raise RuntimeError(f"All Gemini models failed: {last_error}")
|
publisher/bulk.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
bulk.py
|
| 3 |
+
V9 Autonomous Publisher Engine
|
| 4 |
+
|
| 5 |
+
Purpose:
|
| 6 |
+
--------
|
| 7 |
+
Handles BULK publishing across multiple platforms.
|
| 8 |
+
|
| 9 |
+
Supports:
|
| 10 |
+
- TikTok
|
| 11 |
+
- Reels (Instagram)
|
| 12 |
+
- YouTube Shorts
|
| 13 |
+
- Facebook
|
| 14 |
+
- Any future platform adapter
|
| 15 |
+
|
| 16 |
+
Design:
|
| 17 |
+
-------
|
| 18 |
+
Input -> Normalize -> Dispatch -> Execute -> Collect Results
|
| 19 |
+
|
| 20 |
+
Production Features:
|
| 21 |
+
--------------------
|
| 22 |
+
✔ async concurrency
|
| 23 |
+
✔ retry system
|
| 24 |
+
✔ per-platform isolation
|
| 25 |
+
✔ failure tolerance
|
| 26 |
+
✔ structured logging
|
| 27 |
+
✔ scheduler-compatible
|
| 28 |
+
✔ autonomous engine ready
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
import asyncio
|
| 32 |
+
import traceback
|
| 33 |
+
from typing import Dict, List, Any
|
| 34 |
+
|
| 35 |
+
# Platform adapters
|
| 36 |
+
from publisher.platforms.tiktok import publish_tiktok
|
| 37 |
+
from publisher.platforms.reels import publish_reels
|
| 38 |
+
from publisher.platforms.shorts import publish_shorts
|
| 39 |
+
from publisher.platforms.facebook import publish_facebook
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# =====================================================
|
| 43 |
+
# PLATFORM REGISTRY
|
| 44 |
+
# =====================================================
|
| 45 |
+
|
| 46 |
+
PLATFORM_MAP = {
|
| 47 |
+
"tiktok": publish_tiktok,
|
| 48 |
+
"reels": publish_reels,
|
| 49 |
+
"shorts": publish_shorts,
|
| 50 |
+
"facebook": publish_facebook,
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# =====================================================
|
| 55 |
+
# CONFIG
|
| 56 |
+
# =====================================================
|
| 57 |
+
|
| 58 |
+
MAX_CONCURRENT_POSTS = 5
|
| 59 |
+
MAX_RETRIES = 2
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# =====================================================
|
| 63 |
+
# HELPERS
|
| 64 |
+
# =====================================================
|
| 65 |
+
|
| 66 |
+
async def execute_with_retry(func, payload: Dict, retries=MAX_RETRIES):
|
| 67 |
+
"""
|
| 68 |
+
Safe execution wrapper with retries.
|
| 69 |
+
"""
|
| 70 |
+
|
| 71 |
+
attempt = 0
|
| 72 |
+
|
| 73 |
+
while attempt <= retries:
|
| 74 |
+
try:
|
| 75 |
+
result = await func(payload)
|
| 76 |
+
return {
|
| 77 |
+
"status": "success",
|
| 78 |
+
"result": result,
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
except Exception as e:
|
| 82 |
+
attempt += 1
|
| 83 |
+
|
| 84 |
+
if attempt > retries:
|
| 85 |
+
return {
|
| 86 |
+
"status": "failed",
|
| 87 |
+
"error": str(e),
|
| 88 |
+
"trace": traceback.format_exc(),
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
await asyncio.sleep(2)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# =====================================================
|
| 95 |
+
# SINGLE JOB EXECUTOR
|
| 96 |
+
# =====================================================
|
| 97 |
+
|
| 98 |
+
async def process_job(job: Dict[str, Any]):
|
| 99 |
+
"""
|
| 100 |
+
Expected job format:
|
| 101 |
+
|
| 102 |
+
{
|
| 103 |
+
"platform": "tiktok",
|
| 104 |
+
"video_url": "...",
|
| 105 |
+
"caption": "...",
|
| 106 |
+
"hashtags": [],
|
| 107 |
+
"thumbnail": "...",
|
| 108 |
+
"schedule_time": optional
|
| 109 |
+
}
|
| 110 |
+
"""
|
| 111 |
+
|
| 112 |
+
platform = job.get("platform")
|
| 113 |
+
|
| 114 |
+
if platform not in PLATFORM_MAP:
|
| 115 |
+
return {
|
| 116 |
+
"status": "failed",
|
| 117 |
+
"error": f"Unsupported platform: {platform}",
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
publisher = PLATFORM_MAP[platform]
|
| 121 |
+
|
| 122 |
+
return await execute_with_retry(publisher, job)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# =====================================================
|
| 126 |
+
# BULK ENGINE
|
| 127 |
+
# =====================================================
|
| 128 |
+
|
| 129 |
+
async def bulk_publish(jobs: List[Dict[str, Any]]):
|
| 130 |
+
"""
|
| 131 |
+
Main bulk execution engine.
|
| 132 |
+
"""
|
| 133 |
+
|
| 134 |
+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_POSTS)
|
| 135 |
+
|
| 136 |
+
results = []
|
| 137 |
+
|
| 138 |
+
async def limited_job(job):
|
| 139 |
+
async with semaphore:
|
| 140 |
+
return await process_job(job)
|
| 141 |
+
|
| 142 |
+
tasks = [limited_job(job) for job in jobs]
|
| 143 |
+
|
| 144 |
+
completed = await asyncio.gather(*tasks, return_exceptions=False)
|
| 145 |
+
|
| 146 |
+
results.extend(completed)
|
| 147 |
+
|
| 148 |
+
return summarize_results(results)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# =====================================================
|
| 152 |
+
# SUMMARY
|
| 153 |
+
# =====================================================
|
| 154 |
+
|
| 155 |
+
def summarize_results(results: List[Dict]):
|
| 156 |
+
success = sum(1 for r in results if r["status"] == "success")
|
| 157 |
+
failed = len(results) - success
|
| 158 |
+
|
| 159 |
+
return {
|
| 160 |
+
"status": "completed",
|
| 161 |
+
"total_jobs": len(results),
|
| 162 |
+
"successful": success,
|
| 163 |
+
"failed": failed,
|
| 164 |
+
"results": results,
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
# =====================================================
|
| 169 |
+
# FASTAPI ENTRYPOINT
|
| 170 |
+
# =====================================================
|
| 171 |
+
|
| 172 |
+
async def execute(payload: Dict):
|
| 173 |
+
"""
|
| 174 |
+
Universal endpoint handler
|
| 175 |
+
|
| 176 |
+
POST /execute/bulk_publish
|
| 177 |
+
"""
|
| 178 |
+
|
| 179 |
+
jobs = payload.get("jobs")
|
| 180 |
+
|
| 181 |
+
if not jobs:
|
| 182 |
+
return {
|
| 183 |
+
"status": "error",
|
| 184 |
+
"message": "No jobs provided",
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
return await bulk_publish(jobs)
|
publisher/hashtags.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from publisher.ai.gemini_client import get_model
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def generate_hashtags(video_path: str):
|
| 5 |
+
|
| 6 |
+
model = get_model()
|
| 7 |
+
|
| 8 |
+
prompt = """
|
| 9 |
+
Generate 20 viral hashtags for short-form content.
|
| 10 |
+
Return comma-separated only.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
res = model.generate_content(prompt)
|
| 14 |
+
|
| 15 |
+
tags = res.text.replace("\n", "").strip()
|
| 16 |
+
|
| 17 |
+
return {
|
| 18 |
+
"hashtags": tags
|
| 19 |
+
}
|
publisher/metadata_engine.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from publisher.ai.gemini_client import get_model
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def generate_metadata(video_path: str):
|
| 5 |
+
|
| 6 |
+
model = get_model()
|
| 7 |
+
|
| 8 |
+
prompt = f"""
|
| 9 |
+
Generate viral short-form video metadata.
|
| 10 |
+
|
| 11 |
+
Return JSON:
|
| 12 |
+
title
|
| 13 |
+
description
|
| 14 |
+
hook
|
| 15 |
+
audience
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
response = model.generate_content(prompt)
|
| 19 |
+
|
| 20 |
+
text = response.text.strip()
|
| 21 |
+
|
| 22 |
+
return {
|
| 23 |
+
"metadata": text
|
| 24 |
+
}
|
publisher/models.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class PublishPayload(BaseModel):
|
| 6 |
+
|
| 7 |
+
video_path: str
|
| 8 |
+
|
| 9 |
+
title: str
|
| 10 |
+
description: str
|
| 11 |
+
|
| 12 |
+
hashtags: List[str]
|
| 13 |
+
|
| 14 |
+
thumbnail: Optional[str] = None
|
| 15 |
+
schedule_time: Optional[int] = None
|
| 16 |
+
|
| 17 |
+
platforms: List[str]
|
publisher/oauth/providers/google.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/providers/google.py
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
import os
|
| 5 |
+
from ..storage import save_tokens
|
| 6 |
+
|
| 7 |
+
CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID")
|
| 8 |
+
CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET")
|
| 9 |
+
|
| 10 |
+
REDIRECT_URI = "https://your-domain.com/oauth/callback/google"
|
| 11 |
+
|
| 12 |
+
AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth"
|
| 13 |
+
TOKEN_URL = "https://oauth2.googleapis.com/token"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def authorization_url(state):
|
| 17 |
+
|
| 18 |
+
scope = (
|
| 19 |
+
"https://www.googleapis.com/auth/youtube.upload "
|
| 20 |
+
"https://www.googleapis.com/auth/userinfo.profile"
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
return (
|
| 24 |
+
f"{AUTH_URL}"
|
| 25 |
+
f"?client_id={CLIENT_ID}"
|
| 26 |
+
f"&redirect_uri={REDIRECT_URI}"
|
| 27 |
+
f"&response_type=code"
|
| 28 |
+
f"&scope={scope}"
|
| 29 |
+
f"&access_type=offline"
|
| 30 |
+
f"&state={state}"
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
async def exchange_code(code):
|
| 35 |
+
|
| 36 |
+
async with httpx.AsyncClient() as client:
|
| 37 |
+
r = await client.post(
|
| 38 |
+
TOKEN_URL,
|
| 39 |
+
data={
|
| 40 |
+
"code": code,
|
| 41 |
+
"client_id": CLIENT_ID,
|
| 42 |
+
"client_secret": CLIENT_SECRET,
|
| 43 |
+
"redirect_uri": REDIRECT_URI,
|
| 44 |
+
"grant_type": "authorization_code",
|
| 45 |
+
},
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
return r.json()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def store(user_id, tokens):
|
| 52 |
+
save_tokens(user_id, "google", tokens)
|
publisher/oauth/providers/meta.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/providers/meta.py
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
import os
|
| 5 |
+
from ..storage import save_tokens
|
| 6 |
+
|
| 7 |
+
APP_ID = os.getenv("META_APP_ID")
|
| 8 |
+
APP_SECRET = os.getenv("META_APP_SECRET")
|
| 9 |
+
|
| 10 |
+
REDIRECT_URI = "https://your-domain.com/oauth/callback/meta"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def authorization_url(state):
|
| 14 |
+
|
| 15 |
+
return (
|
| 16 |
+
"https://www.facebook.com/v19.0/dialog/oauth"
|
| 17 |
+
f"?client_id={APP_ID}"
|
| 18 |
+
f"&redirect_uri={REDIRECT_URI}"
|
| 19 |
+
"&scope=pages_manage_posts,pages_read_engagement,"
|
| 20 |
+
"instagram_content_publish"
|
| 21 |
+
f"&state={state}"
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
async def exchange_code(code):
|
| 26 |
+
|
| 27 |
+
async with httpx.AsyncClient() as client:
|
| 28 |
+
r = await client.get(
|
| 29 |
+
"https://graph.facebook.com/v19.0/oauth/access_token",
|
| 30 |
+
params={
|
| 31 |
+
"client_id": APP_ID,
|
| 32 |
+
"redirect_uri": REDIRECT_URI,
|
| 33 |
+
"client_secret": APP_SECRET,
|
| 34 |
+
"code": code,
|
| 35 |
+
},
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
return r.json()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def store(user_id, tokens):
|
| 42 |
+
save_tokens(user_id, "meta", tokens)
|
publisher/oauth/providers/tiktok.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/providers/tiktok.py
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
import os
|
| 5 |
+
from ..storage import save_tokens
|
| 6 |
+
|
| 7 |
+
CLIENT_KEY = os.getenv("TIKTOK_CLIENT_KEY")
|
| 8 |
+
CLIENT_SECRET = os.getenv("TIKTOK_CLIENT_SECRET")
|
| 9 |
+
|
| 10 |
+
REDIRECT_URI = "https://your-domain.com/oauth/callback/tiktok"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def authorization_url(state):
|
| 14 |
+
|
| 15 |
+
scope = "user.info.basic,video.upload"
|
| 16 |
+
|
| 17 |
+
return (
|
| 18 |
+
"https://www.tiktok.com/v2/auth/authorize/"
|
| 19 |
+
f"?client_key={CLIENT_KEY}"
|
| 20 |
+
f"&response_type=code"
|
| 21 |
+
f"&scope={scope}"
|
| 22 |
+
f"&redirect_uri={REDIRECT_URI}"
|
| 23 |
+
f"&state={state}"
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
async def exchange_code(code):
|
| 28 |
+
|
| 29 |
+
async with httpx.AsyncClient() as client:
|
| 30 |
+
r = await client.post(
|
| 31 |
+
"https://open.tiktokapis.com/v2/oauth/token/",
|
| 32 |
+
data={
|
| 33 |
+
"client_key": CLIENT_KEY,
|
| 34 |
+
"client_secret": CLIENT_SECRET,
|
| 35 |
+
"code": code,
|
| 36 |
+
"grant_type": "authorization_code",
|
| 37 |
+
"redirect_uri": REDIRECT_URI,
|
| 38 |
+
},
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
return r.json()
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def store(user_id, tokens):
|
| 45 |
+
save_tokens(user_id, "tiktok", tokens)
|
publisher/oauth/providers/youtube.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/providers/youtube.py
|
| 2 |
+
|
| 3 |
+
from .google import authorization_url, exchange_code
|
| 4 |
+
from ..storage import save_tokens
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def store(user_id, tokens):
|
| 8 |
+
save_tokens(user_id, "youtube", tokens)
|
publisher/oauth/router.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/router.py
|
| 2 |
+
|
| 3 |
+
from fastapi import APIRouter, Request
|
| 4 |
+
from fastapi.responses import RedirectResponse
|
| 5 |
+
|
| 6 |
+
from .sessions import create_session, get_user
|
| 7 |
+
from .providers import google, meta, tiktok, youtube
|
| 8 |
+
|
| 9 |
+
router = APIRouter()
|
| 10 |
+
|
| 11 |
+
PROVIDERS = {
|
| 12 |
+
"google": google,
|
| 13 |
+
"meta": meta,
|
| 14 |
+
"tiktok": tiktok,
|
| 15 |
+
"youtube": youtube
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@router.get("/connect/{provider}")
|
| 20 |
+
async def connect(provider: str, user_id: str):
|
| 21 |
+
|
| 22 |
+
if provider not in PROVIDERS:
|
| 23 |
+
return {"error": "provider not supported"}
|
| 24 |
+
|
| 25 |
+
state = create_session(user_id)
|
| 26 |
+
|
| 27 |
+
url = PROVIDERS[provider].authorization_url(state)
|
| 28 |
+
|
| 29 |
+
return RedirectResponse(url)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@router.get("/callback/{provider}")
|
| 33 |
+
async def callback(provider: str, request: Request):
|
| 34 |
+
|
| 35 |
+
if provider not in PROVIDERS:
|
| 36 |
+
return {"error": "provider not supported"}
|
| 37 |
+
|
| 38 |
+
state = request.query_params.get("state")
|
| 39 |
+
code = request.query_params.get("code")
|
| 40 |
+
|
| 41 |
+
user_id = get_user(state)
|
| 42 |
+
|
| 43 |
+
tokens = await PROVIDERS[provider].exchange_code(code)
|
| 44 |
+
|
| 45 |
+
PROVIDERS[provider].store(user_id, tokens)
|
| 46 |
+
|
| 47 |
+
return {"status": f"{provider} connected"}
|
publisher/oauth/sessions.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/sessions.py
|
| 2 |
+
|
| 3 |
+
import secrets
|
| 4 |
+
import time
|
| 5 |
+
|
| 6 |
+
_sessions = {}
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def create_session(user_id):
|
| 10 |
+
state = secrets.token_urlsafe(32)
|
| 11 |
+
_sessions[state] = {
|
| 12 |
+
"user_id": user_id,
|
| 13 |
+
"created": time.time()
|
| 14 |
+
}
|
| 15 |
+
return state
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def get_user(state):
|
| 19 |
+
session = _sessions.get(state)
|
| 20 |
+
if not session:
|
| 21 |
+
return None
|
| 22 |
+
return session["user_id"]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def cleanup(expiry=600):
|
| 26 |
+
now = time.time()
|
| 27 |
+
for k in list(_sessions.keys()):
|
| 28 |
+
if now - _sessions[k]["created"] > expiry:
|
| 29 |
+
del _sessions[k]
|
publisher/oauth/storage.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/oauth/storage.py
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from cryptography.fernet import Fernet
|
| 7 |
+
|
| 8 |
+
STORAGE_DIR = Path("oauth_tokens")
|
| 9 |
+
STORAGE_DIR.mkdir(exist_ok=True)
|
| 10 |
+
|
| 11 |
+
KEY_PATH = STORAGE_DIR / "secret.key"
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def load_key():
|
| 15 |
+
if not KEY_PATH.exists():
|
| 16 |
+
key = Fernet.generate_key()
|
| 17 |
+
KEY_PATH.write_bytes(key)
|
| 18 |
+
return KEY_PATH.read_bytes()
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
fernet = Fernet(load_key())
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _file(user_id, provider):
|
| 25 |
+
return STORAGE_DIR / f"{user_id}_{provider}.json"
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def save_tokens(user_id: str, provider: str, data: dict):
|
| 29 |
+
encrypted = fernet.encrypt(json.dumps(data).encode())
|
| 30 |
+
_file(user_id, provider).write_bytes(encrypted)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def load_tokens(user_id: str, provider: str):
|
| 34 |
+
f = _file(user_id, provider)
|
| 35 |
+
if not f.exists():
|
| 36 |
+
return None
|
| 37 |
+
decrypted = fernet.decrypt(f.read_bytes())
|
| 38 |
+
return json.loads(decrypted.decode())
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def delete_tokens(user_id: str, provider: str):
|
| 42 |
+
f = _file(user_id, provider)
|
| 43 |
+
if f.exists():
|
| 44 |
+
f.unlink()
|
publisher/platform_dispatcher.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from importlib import import_module
|
| 2 |
+
import logging
|
| 3 |
+
|
| 4 |
+
logger = logging.getLogger("platform-dispatcher")
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def _safe_import(module_path, fn_name):
|
| 8 |
+
try:
|
| 9 |
+
module = import_module(module_path)
|
| 10 |
+
return getattr(module, fn_name)
|
| 11 |
+
except Exception as e:
|
| 12 |
+
logger.warning(f"[Dispatcher] Missing {module_path}: {str(e)}")
|
| 13 |
+
return None
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
# lazy-loaded publishers (prevents boot crash)
|
| 17 |
+
|
| 18 |
+
def dispatch_publish(video_path=None, payload=None, variants=None):
|
| 19 |
+
|
| 20 |
+
payload = payload or {}
|
| 21 |
+
results = {}
|
| 22 |
+
|
| 23 |
+
youtube = _safe_import("publisher.platforms.youtube", "publish_youtube")
|
| 24 |
+
tiktok = _safe_import("publisher.platforms.tiktok", "publish_tiktok")
|
| 25 |
+
reels = _safe_import("publisher.platforms.reels", "publish_reels")
|
| 26 |
+
shorts = _safe_import("publisher.platforms.shorts", "publish_shorts")
|
| 27 |
+
facebook = _safe_import("publisher.platforms.facebook", "publish_facebook")
|
| 28 |
+
|
| 29 |
+
async def run():
|
| 30 |
+
|
| 31 |
+
if youtube:
|
| 32 |
+
results["youtube"] = await youtube({"video_path": video_path, **payload})
|
| 33 |
+
|
| 34 |
+
if tiktok:
|
| 35 |
+
results["tiktok"] = await tiktok({"video_path": video_path, **payload})
|
| 36 |
+
|
| 37 |
+
if reels:
|
| 38 |
+
results["reels"] = await reels({"video_path": video_path, **payload})
|
| 39 |
+
|
| 40 |
+
if shorts:
|
| 41 |
+
results["shorts"] = await shorts({"video_path": video_path, **payload})
|
| 42 |
+
|
| 43 |
+
if facebook:
|
| 44 |
+
results["facebook"] = await facebook({"video_path": video_path, **payload})
|
| 45 |
+
|
| 46 |
+
return results
|
| 47 |
+
|
| 48 |
+
return run()
|
publisher/platforms/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Platforms package
|
publisher/platforms/auth.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
# =========================
|
| 4 |
+
# SIMPLE ENV AUTH LOADER
|
| 5 |
+
# =========================
|
| 6 |
+
|
| 7 |
+
def load_env_var(key: str, default=None):
|
| 8 |
+
"""
|
| 9 |
+
Safe environment variable loader.
|
| 10 |
+
Works without python-dotenv dependency.
|
| 11 |
+
"""
|
| 12 |
+
return os.environ.get(key, default)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def get_token(platform: str):
|
| 16 |
+
"""
|
| 17 |
+
Returns stored token for platform.
|
| 18 |
+
No external dependency version.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
key_map = {
|
| 22 |
+
"tiktok": "TIKTOK_TOKEN",
|
| 23 |
+
"youtube": "YOUTUBE_TOKEN",
|
| 24 |
+
"facebook": "FACEBOOK_TOKEN",
|
| 25 |
+
"reels": "META_TOKEN",
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
env_key = key_map.get(platform)
|
| 29 |
+
|
| 30 |
+
if not env_key:
|
| 31 |
+
raise ValueError(f"Unsupported platform: {platform}")
|
| 32 |
+
|
| 33 |
+
token = load_env_var(env_key)
|
| 34 |
+
|
| 35 |
+
if not token:
|
| 36 |
+
raise ValueError(f"Missing token for {platform} ({env_key})")
|
| 37 |
+
|
| 38 |
+
return token
|
publisher/platforms/base.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
def validate_payload(payload):
|
| 4 |
+
|
| 5 |
+
if "video_path" not in payload:
|
| 6 |
+
raise Exception("video_path required")
|
| 7 |
+
|
| 8 |
+
if not os.path.exists(payload["video_path"]):
|
| 9 |
+
raise Exception("Video missing")
|
| 10 |
+
|
| 11 |
+
payload.setdefault("caption", "")
|
| 12 |
+
payload.setdefault("hashtags", [])
|
| 13 |
+
payload.setdefault("thumbnail", None)
|
| 14 |
+
|
| 15 |
+
return payload
|
publisher/platforms/errors.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class PublisherError(Exception):
|
| 2 |
+
pass
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class PlatformAuthError(PublisherError):
|
| 6 |
+
pass
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class PlatformUploadError(PublisherError):
|
| 10 |
+
pass
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class PlatformRateLimit(PublisherError):
|
| 14 |
+
pass
|
publisher/platforms/facebook.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# publisher/platforms/facebook.py
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
from .auth import get_token
|
| 5 |
+
from .base import validate_payload
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
async def publish_facebook(payload):
|
| 9 |
+
|
| 10 |
+
payload = validate_payload(payload)
|
| 11 |
+
|
| 12 |
+
page_id = get_token("FACEBOOK_PAGE_ID")
|
| 13 |
+
token = get_token("META_ACCESS_TOKEN")
|
| 14 |
+
|
| 15 |
+
async with httpx.AsyncClient(timeout=600) as client:
|
| 16 |
+
|
| 17 |
+
with open(payload["video_path"], "rb") as f:
|
| 18 |
+
|
| 19 |
+
res = await client.post(
|
| 20 |
+
f"https://graph-video.facebook.com/{page_id}/videos",
|
| 21 |
+
data={
|
| 22 |
+
"description": payload["caption"],
|
| 23 |
+
"access_token": token,
|
| 24 |
+
},
|
| 25 |
+
files={"source": f},
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
return {"platform": "facebook", "status": "published"}
|
publisher/platforms/http.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import asyncio
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
async def request(
|
| 6 |
+
method,
|
| 7 |
+
url,
|
| 8 |
+
headers=None,
|
| 9 |
+
data=None,
|
| 10 |
+
json=None,
|
| 11 |
+
files=None,
|
| 12 |
+
retries=3,
|
| 13 |
+
):
|
| 14 |
+
|
| 15 |
+
for attempt in range(retries):
|
| 16 |
+
|
| 17 |
+
try:
|
| 18 |
+
async with httpx.AsyncClient(timeout=120) as client:
|
| 19 |
+
|
| 20 |
+
r = await client.request(
|
| 21 |
+
method,
|
| 22 |
+
url,
|
| 23 |
+
headers=headers,
|
| 24 |
+
data=data,
|
| 25 |
+
json=json,
|
| 26 |
+
files=files,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
r.raise_for_status()
|
| 30 |
+
|
| 31 |
+
return r.json()
|
| 32 |
+
|
| 33 |
+
except Exception as e:
|
| 34 |
+
|
| 35 |
+
if attempt == retries - 1:
|
| 36 |
+
raise
|
| 37 |
+
|
| 38 |
+
await asyncio.sleep(2 ** attempt)
|