.github/scripts/compile_binaries.py

#!/usr/bin/env python3
"""
DoppelGen Hardened Native Compiler Script
Compiles core Python packages into Cython native C-extensions (.so)
with fallback to bytecode (.pyc) for dynamic modules, and strips raw .py source files.
"""

import os
import sys
import shutil
import py_compile
import subprocess
from setuptools import setup, Extension
from Cython.Build import cythonize

def main():
    print("=" * 60)
    print("🚀 Starting DoppelGen Cython Native Binary Compilation (.so)...")
    print("=" * 60)

    base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
    os.chdir(base_dir)

    target_packages = ["doppelgen", "actora", "voxa", "sonora", "scenea", "captiona", "ocula"]

    compiled_so_count = 0
    compiled_pyc_count = 0
    deleted_count = 0

    for pkg in target_packages:
        pkg_dir = os.path.join(base_dir, pkg)
        if not os.path.exists(pkg_dir):
            continue

        os.chdir(pkg_dir)

        py_files_in_pkg = []
        for root, dirs, files in os.walk(pkg_dir):
            if "third_party" in root or "hf_cache" in root or ".cache" in root or "checkpoints" in root:
                continue
            if os.path.basename(root) in ["runners", "test", "tests", "docker", "build", "__pycache__"]:
                continue
            for file in files:
                if file.endswith(".py") and not file.startswith("."):
                    if file in ["__init__.py", "preload_models.py", "app.py", "setup.py"]:
                        continue
                    full_path = os.path.join(root, file)
                    rel_to_pkg = os.path.relpath(full_path, pkg_dir)
                    module_name = rel_to_pkg[:-3].replace(os.sep, ".")
                    py_files_in_pkg.append((full_path, rel_to_pkg, module_name))

        print(f"📦 Compiling {len(py_files_in_pkg)} modules in {pkg}...")

        for full_path, rel_to_pkg, module_name in py_files_in_pkg:
            ext = Extension(
                module_name,
                sources=[rel_to_pkg],
                extra_compile_args=["-O3", "-fPIC", "-Wno-unused-variable"],
            )

            success = False
            try:
                setup(
                    ext_modules=cythonize(
                        [ext],
                        compiler_directives={
                            'language_level': "3",
                            'always_allow_keywords': True,
                            'embedsignature': False,
                            'annotation_typing': False,
                        },
                        quiet=True,
                    ),
                    script_args=["build_ext", "--inplace"],
                )
                dir_name = os.path.dirname(rel_to_pkg)
                base_name = os.path.basename(rel_to_pkg)[:-3]
                search_dir = os.path.join(pkg_dir, dir_name) if dir_name else pkg_dir
                matching_so = [f for f in os.listdir(search_dir) if f.startswith(base_name) and f.endswith(".so")]
                if matching_so:
                    success = True
                    compiled_so_count += 1
            except Exception as e:
                print(f"⚠️ Cython compilation fallback to .pyc for {rel_to_pkg}: {e}")

            if not success:
                pyc_path = full_path + "c"
                try:
                    py_compile.compile(full_path, cfile=pyc_path, doraise=True, optimize=2)
                    compiled_pyc_count += 1
                except Exception as pe:
                    print(f"❌ Bytecode compilation failed for {rel_to_pkg}: {pe}")

        for full_path, rel_to_pkg, module_name in py_files_in_pkg:
            dir_name = os.path.dirname(rel_to_pkg)
            base_name = os.path.basename(rel_to_pkg)[:-3]
            search_dir = os.path.join(pkg_dir, dir_name) if dir_name else pkg_dir
            matching_so = [f for f in os.listdir(search_dir) if f.startswith(base_name) and f.endswith(".so")] if os.path.exists(search_dir) else []
            if matching_so:
                if os.path.exists(full_path):
                    os.remove(full_path)
                    deleted_count += 1
            else:
                print(f"ℹ️ Preserved {rel_to_pkg} as .py source (no .so generated)", flush=True)

    os.chdir(base_dir)

    # Strip debugging symbols from generated .so files
    for root, dirs, files in os.walk(base_dir):
        if ".git" in root or "venv" in root or ".venv" in root:
            continue
        for file in files:
            if file.endswith(".so"):
                so_path = os.path.join(root, file)
                try:
                    subprocess.run(["strip", "--strip-debug", so_path], check=False)
                except Exception:
                    pass

    print(f"🔒 Compiled {compiled_so_count} Cython .so C-extensions (stripped symbols).")
    print(f"⚡ Compiled {compiled_pyc_count} modules to optimized .pyc bytecode.")

    # Clean up temporary Cython .c, .cpp, and build directories
    for root, dirs, files in os.walk(base_dir):
        for file in files:
            if file.endswith(".c") or file.endswith(".cpp"):
                if "third_party" not in root and not file.startswith("c_"):
                    c_path = os.path.join(root, file)
                    try:
                        os.remove(c_path)
                    except OSError:
                        pass

        build_dir = os.path.join(root, "build")
        if os.path.exists(build_dir):
            shutil.rmtree(build_dir, ignore_errors=True)

    print(f"🧹 Removed {deleted_count} original source .py files (replaced with compiled binaries).")
    print("=" * 60)
    print("✨ DoppelGen Hardened Binary Build Complete!")
    print("=" * 60)

if __name__ == "__main__":
    main()