import os import subprocess import sys import warp from warp.build_dll import * # set build output path off this file base_path = os.path.dirname(os.path.realpath(__file__)) build_path = os.path.join(base_path, "warp") llvm_project_path = os.path.join(base_path, "external/llvm-project") llvm_build_path = os.path.join(llvm_project_path, "out/build/") llvm_install_path = os.path.join(llvm_project_path, "out/install/") # Fetch prebuilt Clang/LLVM libraries def fetch_prebuilt_libraries(): if os.name == "nt": packman = "tools\\packman\\packman.cmd" packages = {"x86_64": "15.0.7-windows-x86_64-ptx-vs142"} else: packman = "./tools/packman/packman" if sys.platform == "darwin": packages = { "aarch64": "15.0.7-darwin-aarch64-macos11", "x86_64": "15.0.7-darwin-x86_64-macos11", } else: packages = { "aarch64": "15.0.7-linux-aarch64-gcc7.5", "x86_64": "15.0.7-linux-x86_64-ptx-gcc7.5-cxx11abi0", } for arch in packages: subprocess.check_call( [ packman, "install", "-l", f"./_build/host-deps/llvm-project/release-{arch}", "clang+llvm-warp", packages[arch], ] ) def build_from_source_for_arch(args, arch, llvm_source): # Check out the LLVM project Git repository, unless it already exists if not os.path.exists(llvm_source): # Install dependencies subprocess.check_call([sys.executable, "-m", "pip", "install", "gitpython"]) subprocess.check_call([sys.executable, "-m", "pip", "install", "cmake"]) subprocess.check_call([sys.executable, "-m", "pip", "install", "ninja"]) from git import Repo repo_url = "https://github.com/llvm/llvm-project.git" print(f"Cloning LLVM project from {repo_url}...") shallow_clone = True # https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ if shallow_clone: repo = Repo.clone_from(repo_url, to_path=llvm_source, single_branch=True, branch="llvmorg-15.0.7", depth=1) else: repo = Repo.clone_from(repo_url, to_path=llvm_source) repo.git.checkout("tags/llvmorg-15.0.7", "-b", "llvm-15.0.7") print(f"Using LLVM project source from {llvm_source}") # CMake supports Debug, Release, RelWithDebInfo, and MinSizeRel builds if warp.config.mode == "release": # prefer smaller size over aggressive speed cmake_build_type = "MinSizeRel" else: # When warp.config.mode == "debug" we build a Debug version of warp.dll but # we generally don't want warp-clang.dll to be a slow Debug version. if args.debug_llvm: cmake_build_type = "Debug" else: # The GDB/LLDB debugger observes the __jit_debug_register_code symbol # defined by the LLVM JIT, for which it needs debug info. cmake_build_type = "RelWithDebInfo" # Location of cmake and ninja installed through pip (see build.bat / build.sh) python_bin = "python/Scripts" if sys.platform == "win32" else "python/bin" os.environ["PATH"] = os.path.join(base_path, "_build/target-deps/" + python_bin) + os.pathsep + os.environ["PATH"] if arch == "aarch64": target_backend = "AArch64" else: target_backend = "X86" if sys.platform == "darwin": host_triple = f"{arch}-apple-macos11" osx_architectures = arch # build one architecture only elif os.name == "nt": host_triple = f"{arch}-pc-windows" osx_architectures = "" else: host_triple = f"{arch}-pc-linux" osx_architectures = "" llvm_path = os.path.join(llvm_source, "llvm") build_path = os.path.join(llvm_build_path, f"{warp.config.mode}-{arch}") install_path = os.path.join(llvm_install_path, f"{warp.config.mode}-{arch}") # Build LLVM and Clang cmake_gen = [ # fmt: off "cmake", "-S", llvm_path, "-B", build_path, "-G", "Ninja", "-D", f"CMAKE_BUILD_TYPE={cmake_build_type}", "-D", "LLVM_USE_CRT_RELEASE=MT", "-D", "LLVM_USE_CRT_MINSIZEREL=MT", "-D", "LLVM_USE_CRT_DEBUG=MTd", "-D", "LLVM_USE_CRT_RELWITHDEBINFO=MTd", "-D", f"LLVM_TARGETS_TO_BUILD={target_backend};NVPTX", "-D", "LLVM_ENABLE_PROJECTS=clang", "-D", "LLVM_ENABLE_ZLIB=FALSE", "-D", "LLVM_ENABLE_ZSTD=FALSE", "-D", "LLVM_ENABLE_TERMINFO=FALSE", "-D", "LLVM_BUILD_LLVM_C_DYLIB=FALSE", "-D", "LLVM_BUILD_RUNTIME=FALSE", "-D", "LLVM_BUILD_RUNTIMES=FALSE", "-D", "LLVM_BUILD_TOOLS=FALSE", "-D", "LLVM_BUILD_UTILS=FALSE", "-D", "LLVM_INCLUDE_BENCHMARKS=FALSE", "-D", "LLVM_INCLUDE_DOCS=FALSE", "-D", "LLVM_INCLUDE_EXAMPLES=FALSE", "-D", "LLVM_INCLUDE_RUNTIMES=FALSE", "-D", "LLVM_INCLUDE_TESTS=FALSE", "-D", "LLVM_INCLUDE_TOOLS=TRUE", # Needed by Clang "-D", "LLVM_INCLUDE_UTILS=FALSE", "-D", "CMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0", # The pre-C++11 ABI is still the default on the CentOS 7 toolchain "-D", f"CMAKE_INSTALL_PREFIX={install_path}", "-D", f"LLVM_HOST_TRIPLE={host_triple}", "-D", f"CMAKE_OSX_ARCHITECTURES={osx_architectures}", # Disable unused tools and features "-D", "CLANG_BUILD_TOOLS=FALSE", "-D", "LLVM_ENABLE_PLUGINS=FALSE", "-D", "CLANG_PLUGIN_SUPPORT=FALSE", "-D", "CLANG_ENABLE_ARCMT=FALSE", "-D", "CLANG_ENABLE_STATIC_ANALYZER=FALSE", "-D", "CLANG_TOOLING_BUILD_AST_INTROSPECTION=FALSE", "-D", "CLANG_TOOL_AMDGPU_ARCH_BUILD=FALSE", "-D", "CLANG_TOOL_APINOTES_TEST_BUILD=FALSE", "-D", "CLANG_TOOL_ARCMT_TEST_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_CHECK_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_DIFF_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_EXTDEF_MAPPING_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_FORMAT_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_FORMAT_VS_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_FUZZER_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_IMPORT_TEST_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_LINKER_WRAPPER_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_NVLINK_WRAPPER_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_OFFLOAD_BUNDLER_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_OFFLOAD_PACKAGER_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_OFFLOAD_WRAPPER_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_REFACTOR_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_RENAME_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_REPL_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_SCAN_DEPS_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_SHLIB_BUILD=FALSE", "-D", "CLANG_TOOL_C_ARCMT_TEST_BUILD=FALSE", "-D", "CLANG_TOOL_C_INDEX_TEST_BUILD=FALSE", "-D", "CLANG_TOOL_DIAGTOOL_BUILD=FALSE", "-D", "CLANG_TOOL_DRIVER_BUILD=FALSE", "-D", "CLANG_TOOL_LIBCLANG_BUILD=FALSE", "-D", "CLANG_TOOL_SCAN_BUILD_BUILD=FALSE", "-D", "CLANG_TOOL_SCAN_BUILD_PY_BUILD=FALSE", "-D", "CLANG_TOOL_CLANG_OFFLOAD_BUNDLER_BUILD=FALSE", "-D", "CLANG_TOOL_SCAN_VIEW_BUILD=FALSE", "-D", "LLVM_ENABLE_BINDINGS=FALSE", "-D", "LLVM_ENABLE_OCAMLDOC=FALSE", "-D", "LLVM_TOOL_BUGPOINT_BUILD=FALSE", "-D", "LLVM_TOOL_BUGPOINT_PASSES_BUILD=FALSE", "-D", "LLVM_TOOL_CLANG_BUILD=FALSE", "-D", "LLVM_TOOL_DSYMUTIL_BUILD=FALSE", "-D", "LLVM_TOOL_DXIL_DIS_BUILD=FALSE", "-D", "LLVM_TOOL_GOLD_BUILD=FALSE", "-D", "LLVM_TOOL_LLC_BUILD=FALSE", "-D", "LLVM_TOOL_LLDB_BUILD=FALSE", "-D", "LLVM_TOOL_LLI_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_AR_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_AS_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_AS_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_BCANALYZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CAT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CFI_VERIFY_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CONFIG_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_COV_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CVTRES_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CXXDUMP_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CXXFILT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_CXXMAP_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_C_TEST_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DEBUGINFOD_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DIFF_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DIS_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DIS_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DWARFDUMP_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DWARFUTIL_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_DWP_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_EXEGESIS_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_EXTRACT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_GO_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_GSYMUTIL_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_IFS_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_JITLINK_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_JITLISTENER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_LINK_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_LIPO_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_LTO2_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_LTO_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MCA_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MC_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_ML_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MODEXTRACT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_MT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_NM_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_OBJCOPY_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_OBJDUMP_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_OPT_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_OPT_REPORT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_PDBUTIL_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_PROFDATA_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_PROFGEN_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_RC_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_READOBJ_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_REDUCE_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_REMARK_SIZE_DIFF_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_RTDYLD_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_SHLIB_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_SIM_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_SIZE_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_SPLIT_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_STRESS_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_STRINGS_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_SYMBOLIZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_TAPI_DIFF_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_TLI_CHECKER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_UNDNAME_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_XRAY_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_LTO_BUILD=FALSE", "-D", "LLVM_TOOL_OBJ2YAML_BUILD=FALSE", "-D", "LLVM_TOOL_OPT_BUILD=FALSE", "-D", "LLVM_TOOL_OPT_VIEWER_BUILD=FALSE", "-D", "LLVM_TOOL_REMARKS_SHLIB_BUILD=FALSE", "-D", "LLVM_TOOL_SANCOV_BUILD=FALSE", "-D", "LLVM_TOOL_SANSTATS_BUILD=FALSE", "-D", "LLVM_TOOL_SPLIT_FILE_BUILD=FALSE", "-D", "LLVM_TOOL_VERIFY_USELISTORDER_BUILD=FALSE", "-D", "LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD=FALSE", "-D", "LLVM_TOOL_XCODE_TOOLCHAIN_BUILD=FALSE", "-D", "LLVM_TOOL_YAML2OBJ_BUILD=FALSE", # fmt: on ] subprocess.check_call(cmake_gen, stderr=subprocess.STDOUT) cmake_build = ["cmake", "--build", build_path] subprocess.check_call(cmake_build, stderr=subprocess.STDOUT) cmake_install = ["cmake", "--install", build_path] subprocess.check_call(cmake_install, stderr=subprocess.STDOUT) def build_from_source(args): print("Building Clang/LLVM from source...") if args.llvm_source_path is not None: llvm_source = args.llvm_source_path else: llvm_source = llvm_project_path # build for the machine's architecture build_from_source_for_arch(args, machine_architecture(), llvm_source) # for Apple systems also cross-compile for building a universal binary if sys.platform == "darwin": if machine_architecture() == "x86_64": build_from_source_for_arch(args, "aarch64", llvm_source) else: build_from_source_for_arch(args, "x86_64", llvm_source) # build warp-clang.dll def build_warp_clang_for_arch(args, lib_name, arch): try: cpp_sources = [ "native/clang/clang.cpp", "native/crt.cpp", ] clang_cpp_paths = [os.path.join(build_path, cpp) for cpp in cpp_sources] clang_dll_path = os.path.join(build_path, f"bin/{lib_name}") if args.build_llvm: # obtain Clang and LLVM libraries from the local build install_path = os.path.join(llvm_install_path, f"{warp.config.mode}-{arch}") libpath = os.path.join(install_path, "lib") else: # obtain Clang and LLVM libraries from packman fetch_prebuilt_libraries() libpath = os.path.join(base_path, f"_build/host-deps/llvm-project/release-{arch}/lib") for _, _, libs in os.walk(libpath): break # just the top level contains library files if os.name == "nt": libs.append("Version.lib") libs.append(f'/LIBPATH:"{libpath}"') else: libs = [f"-l{lib[3:-2]}" for lib in libs if os.path.splitext(lib)[1] == ".a"] if sys.platform == "darwin": libs += libs # prevents unresolved symbols due to link order else: libs.insert(0, "-Wl,--start-group") libs.append("-Wl,--end-group") libs.append(f"-L{libpath}") libs.append("-lpthread") libs.append("-ldl") build_dll_for_arch( dll_path=clang_dll_path, cpp_paths=clang_cpp_paths, cu_path=None, libs=libs, mode=warp.config.mode if args.build_llvm else "release", arch=arch, verify_fp=warp.config.verify_fp, fast_math=warp.config.fast_math, ) except Exception as e: # output build error print(f"Warp Clang/LLVM build error: {e}") # report error sys.exit(1) def build_warp_clang(args, lib_name): if sys.platform == "darwin": # create a universal binary by combining x86-64 and AArch64 builds build_warp_clang_for_arch(args, lib_name + "-x86_64", "x86_64") build_warp_clang_for_arch(args, lib_name + "-aarch64", "aarch64") dylib_path = os.path.join(build_path, f"bin/{lib_name}") run_cmd(f"lipo -create -output {dylib_path} {dylib_path}-x86_64 {dylib_path}-aarch64") os.remove(f"{dylib_path}-x86_64") os.remove(f"{dylib_path}-aarch64") else: build_warp_clang_for_arch(args, lib_name, machine_architecture())