#!/usr/bin/env python

# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import codecs
import os
import subprocess
import sys
import distutils.sysconfig
import pybind11

from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext

HERE = os.path.dirname(os.path.realpath(__file__))

def read(*parts):
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read()

class CMakeBuildExt(build_ext):
    def build_extensions(self):

        cmake_python_library = "{}/{}".format(
            distutils.sysconfig.get_config_var("LIBDIR"),
            distutils.sysconfig.get_config_var("INSTSONAME"),
        )
        cmake_python_include_dir = distutils.sysconfig.get_python_inc()

        install_dir = os.path.abspath(
            os.path.dirname(self.get_ext_fullpath("dummy"))
        )
        os.makedirs(install_dir, exist_ok=True)
        cmake_args = [
            "-DCMAKE_INSTALL_PREFIX={}".format(install_dir),
            "-DPython_EXECUTABLE={}".format(sys.executable),
            "-DPython_LIBRARIES={}".format(cmake_python_library),
            "-DPython_INCLUDE_DIRS={}".format(cmake_python_include_dir),
            "-DCMAKE_CUDA_ARCHITECTURES=70;80;90",
            "-DNVSHMEM_HOME={}".format(HERE + "/nvshmem"),
            "-DCUFFTMP_HOME={}".format(HERE + "/cufftmp"),
            "-DCMAKE_BUILD_TYPE={}".format(
                "Debug" if self.debug else "Release"
            ),
            "-DCMAKE_PREFIX_PATH={}".format(pybind11.get_cmake_dir()),
        ]

        os.makedirs(self.build_temp, exist_ok=True)
        subprocess.check_call(
            ["cmake", f"{HERE}/src/cufftmp_jax/"] + cmake_args, cwd=self.build_temp
        )

        # Build all the extensions
        super().build_extensions()

        # Finally run install
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", "install"],
            cwd=self.build_temp,
        )

    def build_extension(self, ext):
        target_name = ext.name.split(".")[-1]
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", target_name],
            cwd=self.build_temp,
        )

extensions = [
    Extension(
        "cufftmp_jax.gpu_ops",
        [
            "src/cufftmp_jax/src/gpu_ops.cpp",
            "src/cufftmp_jax/src/kernels.cu",
        ],
    )]

setup(
    name="fft_jax",
    version='0.0.2',
    author="Leopold Cambier, Zan Xu",
    author_email="lcambier@nvidia.com, zanx@nvidia.com",
    license="All rights reserved",
    description=("FFT + JAX"),
    long_description=read("README.md"),
    long_description_content_type="text/markdown",
    packages=find_packages("src"),
    package_dir={"": "src"},
    include_package_data=True,
    install_requires=[
        # We remove the following JAX dependencies because we base our container from jax toolbox.
        # This avoids unnecessary update/downgrade of jax/jaxlib against the jax/jaxlib in container.
        # For building in other containers/environments, please uncomment the following.
        # "jax[cuda]",
        # "jaxlib"
        ],
    ext_modules=extensions,
    cmdclass={"build_ext": CMakeBuildExt},
)


read

#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import codecs
import os
import subprocess
import sys
import distutils.sysconfig
import pybind11

from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext

HERE = os.path.dirname(os.path.realpath(__file__))

def read(*parts):
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read()

class CMakeBuildExt(build_ext):
    def build_extensions(self):

        cmake_python_library = "{}/{}".format(
            distutils.sysconfig.get_config_var("LIBDIR"),
            distutils.sysconfig.get_config_var("INSTSONAME"),
        )
        cmake_python_include_dir = distutils.sysconfig.get_python_inc()

        install_dir = os.path.abspath(
            os.path.dirname(self.get_ext_fullpath("dummy"))
        )
        os.makedirs(install_dir, exist_ok=True)
        cmake_args = [
            "-DCMAKE_INSTALL_PREFIX={}".format(install_dir),
            "-DPython_EXECUTABLE={}".format(sys.executable),
            "-DPython_LIBRARIES={}".format(cmake_python_library),
            "-DPython_INCLUDE_DIRS={}".format(cmake_python_include_dir),
            "-DCMAKE_CUDA_ARCHITECTURES=70;80;90",


CMakeBuildExt

# See the License for the specific language governing permissions and
# limitations under the License.

import codecs
import os
import subprocess
import sys
import distutils.sysconfig
import pybind11

from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext

HERE = os.path.dirname(os.path.realpath(__file__))

def read(*parts):
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read()

class CMakeBuildExt(build_ext):
    def build_extensions(self):

        cmake_python_library = "{}/{}".format(
            distutils.sysconfig.get_config_var("LIBDIR"),
            distutils.sysconfig.get_config_var("INSTSONAME"),
        )
        cmake_python_include_dir = distutils.sysconfig.get_python_inc()

        install_dir = os.path.abspath(
            os.path.dirname(self.get_ext_fullpath("dummy"))
        )
        os.makedirs(install_dir, exist_ok=True)
        cmake_args = [
            "-DCMAKE_INSTALL_PREFIX={}".format(install_dir),
            "-DPython_EXECUTABLE={}".format(sys.executable),
            "-DPython_LIBRARIES={}".format(cmake_python_library),
            "-DPython_INCLUDE_DIRS={}".format(cmake_python_include_dir),
            "-DCMAKE_CUDA_ARCHITECTURES=70;80;90",
            "-DNVSHMEM_HOME={}".format(HERE + "/nvshmem"),
            "-DCUFFTMP_HOME={}".format(HERE + "/cufftmp"),
            "-DCMAKE_BUILD_TYPE={}".format(
                "Debug" if self.debug else "Release"
            ),
            "-DCMAKE_PREFIX_PATH={}".format(pybind11.get_cmake_dir()),
        ]

        os.makedirs(self.build_temp, exist_ok=True)
        subprocess.check_call(
            ["cmake", f"{HERE}/src/cufftmp_jax/"] + cmake_args, cwd=self.build_temp
        )

        # Build all the extensions
        super().build_extensions()

        # Finally run install
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", "install"],
            cwd=self.build_temp,
        )

    def build_extension(self, ext):
        target_name = ext.name.split(".")[-1]
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", target_name],
            cwd=self.build_temp,
        )

extensions = [
    Extension(
        "cufftmp_jax.gpu_ops",
        [
            "src/cufftmp_jax/src/gpu_ops.cpp",
            "src/cufftmp_jax/src/kernels.cu",
        ],
    )]

setup(
    name="fft_jax",
    version='0.0.2',
    author="Leopold Cambier, Zan Xu",
    author_email="lcambier@nvidia.com, zanx@nvidia.com",
    license="All rights reserved",
    description=("FFT + JAX"),
    long_description=read("README.md"),


# limitations under the License.

import codecs
import os
import subprocess
import sys
import distutils.sysconfig
import pybind11

from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext

HERE = os.path.dirname(os.path.realpath(__file__))

def read(*parts):
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read()

class CMakeBuildExt(build_ext):
    def build_extensions(self):

        cmake_python_library = "{}/{}".format(
            distutils.sysconfig.get_config_var("LIBDIR"),
            distutils.sysconfig.get_config_var("INSTSONAME"),
        )
        cmake_python_include_dir = distutils.sysconfig.get_python_inc()

        install_dir = os.path.abspath(
            os.path.dirname(self.get_ext_fullpath("dummy"))
        )
        os.makedirs(install_dir, exist_ok=True)
        cmake_args = [
            "-DCMAKE_INSTALL_PREFIX={}".format(install_dir),
            "-DPython_EXECUTABLE={}".format(sys.executable),
            "-DPython_LIBRARIES={}".format(cmake_python_library),
            "-DPython_INCLUDE_DIRS={}".format(cmake_python_include_dir),
            "-DCMAKE_CUDA_ARCHITECTURES=70;80;90",
            "-DNVSHMEM_HOME={}".format(HERE + "/nvshmem"),
            "-DCUFFTMP_HOME={}".format(HERE + "/cufftmp"),
            "-DCMAKE_BUILD_TYPE={}".format(
                "Debug" if self.debug else "Release"
            ),
            "-DCMAKE_PREFIX_PATH={}".format(pybind11.get_cmake_dir()),
        ]

        os.makedirs(self.build_temp, exist_ok=True)
        subprocess.check_call(
            ["cmake", f"{HERE}/src/cufftmp_jax/"] + cmake_args, cwd=self.build_temp
        )

        # Build all the extensions
        super().build_extensions()

        # Finally run install
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", "install"],
            cwd=self.build_temp,
        )

    def build_extension(self, ext):
        target_name = ext.name.split(".")[-1]
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", target_name],
            cwd=self.build_temp,
        )

extensions = [
    Extension(
        "cufftmp_jax.gpu_ops",
        [
            "src/cufftmp_jax/src/gpu_ops.cpp",
            "src/cufftmp_jax/src/kernels.cu",
        ],
    )]

setup(


            "-DCMAKE_BUILD_TYPE={}".format(
                "Debug" if self.debug else "Release"
            ),
            "-DCMAKE_PREFIX_PATH={}".format(pybind11.get_cmake_dir()),
        ]

        os.makedirs(self.build_temp, exist_ok=True)
        subprocess.check_call(
            ["cmake", f"{HERE}/src/cufftmp_jax/"] + cmake_args, cwd=self.build_temp
        )

        # Build all the extensions
        super().build_extensions()

        # Finally run install
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", "install"],
            cwd=self.build_temp,
        )

    def build_extension(self, ext):
        target_name = ext.name.split(".")[-1]
        subprocess.check_call(
            ["cmake", "--build", ".", "--target", target_name],
            cwd=self.build_temp,
        )

extensions = [
    Extension(
        "cufftmp_jax.gpu_ops",
        [
            "src/cufftmp_jax/src/gpu_ops.cpp",
            "src/cufftmp_jax/src/kernels.cu",
        ],
    )]

setup(
    name="fft_jax",
    version='0.0.2',
    author="Leopold Cambier, Zan Xu",
    author_email="lcambier@nvidia.com, zanx@nvidia.com",
    license="All rights reserved",
    description=("FFT + JAX"),
    long_description=read("README.md"),


# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

import jax
from jax.experimental.shard_map import shard_map

def Frob_error_impl(dtest, dref):
    derr2 = jax.numpy.linalg.norm(dtest - dref) ** 2
    dnorm2 = jax.numpy.linalg.norm(dref) ** 2
    derr2_sum = jax.lax.psum(derr2,  axis_name="gpus")
    dnorm2_sum = jax.lax.psum(dnorm2, axis_name="gpus")
    error = jax.numpy.sqrt(derr2_sum / dnorm2_sum)
    return error

def Frob_error(dtest, dref, mesh, dist):

    """Computes the relative error in the Frobenius norm

    Arguments:
    dtest  -- the test array, sharded along the axis `ngpus`
    dref   -- the reference array, sharded along the axis `ngpus`
    dist   -- the sharding of dtest and dref
              Should be an instance of fft_common.Dist

    Returns the relative error in the Frobenius norm, i.e.,
    ||dtest - dref||_F / ||dref||_F
    """

    return jax.jit(
                shard_map(
                    Frob_error_impl,
                    mesh=mesh,
                    in_specs=dist.part_spec,
                    out_specs=jax.sharding.PartitionSpec(),
                ),
                in_shardings=jax.sharding.NamedSharding(mesh, dist.part_spec),
                out_shardings=jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
            )(dtest, dref)

def parser():
    parser = argparse.ArgumentParser(
        description="Test program for distributed FFTs in JAX"
    )
    parser.add_argument(
        "implementation",
        type=str,
        choices=['cufftmp', 'xfft'],
        default='cufftmp',
        help='uses cuFFTMp or jit+shard_map'
    )
    parser.add_argument(
        "mode",
        type=str,
        choices=['test', 'perf'],
        default='test',
        help='test (correctness) or perf (performance)'
    )
    parser.add_argument(
        "-x", "--xsize",
        type=int,
        help="Size along X",
        default=1
    )
    parser.add_argument(
        "-y", "--ysize",
        type=int,
        help="Size along Y",
        default=1
    )
    parser.add_argument(
        "-z", "--zsize",
        type=int,
        help="Size along Z",
        default=None
    )
    parser.add_argument(
        "-n", "--size",
        type=int,
        help="Size along X, Y and Z (takes precedence over xsize, ysize and zsize",
        default=None
    )
    parser.add_argument(
        "-c", "--cycles",
        type=int,
        help="Cycles to benchmark (perf only)",
        default=10
    )
    parser.add_argument(
        '-v', '--verbose',
        action='count',
        default=0,
        help="Verbosity level (0 = silent, 2 = debug)"
    )
    parser.add_argument(
        '-d', '--dist',
        type=str,
        choices=['X', 'Y'],
        default='X',
        help="Input distribution (X for SLABS_X or Y for SLABS_Y)"
    )
    parser.add_argument(
        "--multiprocess",
        type=str,
        default=None,
        help="If set, should be of the shape `coordinator,num_procs,proc_id` or `bootstrap` (automatic cluster detection)")
    args = parser.parse_args()
    if args.size:
        shape = args.size, args.size, args.size
    else:
        if args.zsize:
            shape = args.xsize, args.ysize, args.zsize
        else:
            shape = args.xsize, args.ysize
    return {'shape': shape, **vars(args)}


# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

import jax
from jax.experimental.shard_map import shard_map

def Frob_error_impl(dtest, dref):
    derr2 = jax.numpy.linalg.norm(dtest - dref) ** 2
    dnorm2 = jax.numpy.linalg.norm(dref) ** 2
    derr2_sum = jax.lax.psum(derr2,  axis_name="gpus")
    dnorm2_sum = jax.lax.psum(dnorm2, axis_name="gpus")
    error = jax.numpy.sqrt(derr2_sum / dnorm2_sum)
    return error

def Frob_error(dtest, dref, mesh, dist):

    """Computes the relative error in the Frobenius norm

    Arguments:
    dtest  -- the test array, sharded along the axis `ngpus`
    dref   -- the reference array, sharded along the axis `ngpus`
    dist   -- the sharding of dtest and dref
              Should be an instance of fft_common.Dist

    Returns the relative error in the Frobenius norm, i.e.,
    ||dtest - dref||_F / ||dref||_F
    """

    return jax.jit(
                shard_map(
                    Frob_error_impl,
                    mesh=mesh,


# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

import jax
from jax.experimental.shard_map import shard_map

def Frob_error_impl(dtest, dref):
    derr2 = jax.numpy.linalg.norm(dtest - dref) ** 2
    dnorm2 = jax.numpy.linalg.norm(dref) ** 2
    derr2_sum = jax.lax.psum(derr2,  axis_name="gpus")
    dnorm2_sum = jax.lax.psum(dnorm2, axis_name="gpus")
    error = jax.numpy.sqrt(derr2_sum / dnorm2_sum)
    return error

def Frob_error(dtest, dref, mesh, dist):

    """Computes the relative error in the Frobenius norm

    Arguments:
    dtest  -- the test array, sharded along the axis `ngpus`
    dref   -- the reference array, sharded along the axis `ngpus`
    dist   -- the sharding of dtest and dref
              Should be an instance of fft_common.Dist

    Returns the relative error in the Frobenius norm, i.e.,
    ||dtest - dref||_F / ||dref||_F
    """

    return jax.jit(
                shard_map(
                    Frob_error_impl,
                    mesh=mesh,
                    in_specs=dist.part_spec,
                    out_specs=jax.sharding.PartitionSpec(),
                ),
                in_shardings=jax.sharding.NamedSharding(mesh, dist.part_spec),
                out_shardings=jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
            )(dtest, dref)

def parser():
    parser = argparse.ArgumentParser(
        description="Test program for distributed FFTs in JAX"
    )
    parser.add_argument(
        "implementation",
        type=str,
        choices=['cufftmp', 'xfft'],
        default='cufftmp',
        help='uses cuFFTMp or jit+shard_map'
    )
    parser.add_argument(
        "mode",
        type=str,
        choices=['test', 'perf'],
        default='test',
        help='test (correctness) or perf (performance)'
    )


parser

    dref   -- the reference array, sharded along the axis `ngpus`
    dist   -- the sharding of dtest and dref
              Should be an instance of fft_common.Dist

    Returns the relative error in the Frobenius norm, i.e.,
    ||dtest - dref||_F / ||dref||_F
    """

    return jax.jit(
                shard_map(
                    Frob_error_impl,
                    mesh=mesh,
                    in_specs=dist.part_spec,
                    out_specs=jax.sharding.PartitionSpec(),
                ),
                in_shardings=jax.sharding.NamedSharding(mesh, dist.part_spec),
                out_shardings=jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
            )(dtest, dref)

def parser():
    parser = argparse.ArgumentParser(
        description="Test program for distributed FFTs in JAX"
    )
    parser.add_argument(
        "implementation",
        type=str,
        choices=['cufftmp', 'xfft'],
        default='cufftmp',
        help='uses cuFFTMp or jit+shard_map'
    )
    parser.add_argument(
        "mode",
        type=str,
        choices=['test', 'perf'],
        default='test',
        help='test (correctness) or perf (performance)'
    )
    parser.add_argument(
        "-x", "--xsize",
        type=int,
        help="Size along X",
        default=1
    )
    parser.add_argument(
        "-y", "--ysize",
        type=int,
        help="Size along Y",
        default=1
    )
    parser.add_argument(
        "-z", "--zsize",
        type=int,
        help="Size along Z",
        default=None
    )
    parser.add_argument(
        "-n", "--size",
        type=int,
        help="Size along X, Y and Z (takes precedence over xsize, ysize and zsize",
        default=None
    )
    parser.add_argument(
        "-c", "--cycles",
        type=int,
        help="Cycles to benchmark (perf only)",
        default=10
    )
    parser.add_argument(
        '-v', '--verbose',
        action='count',
        default=0,
        help="Verbosity level (0 = silent, 2 = debug)"
    )
    parser.add_argument(
        '-d', '--dist',
        type=str,
        choices=['X', 'Y'],
        default='X',
        help="Input distribution (X for SLABS_X or Y for SLABS_Y)"
    )
    parser.add_argument(
        "--multiprocess",
        type=str,
        default=None,
        help="If set, should be of the shape `coordinator,num_procs,proc_id` or `bootstrap` (automatic cluster detection)")
    args = parser.parse_args()
    if args.size:
        shape = args.size, args.size, args.size
    else:
        if args.zsize:
            shape = args.xsize, args.ysize, args.zsize
        else:
            shape = args.xsize, args.ysize
    return {'shape': shape, **vars(args)}


# -*- coding: utf-8 -*-

# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time
import math
import numpy as np
import sys

import jax
import jax.numpy as jnp
from jax.sharding import Mesh

from fft_common import Dist, Dir
from cufftmp_jax import cufftmp
from xfft import xfft

import helpers

def main():

    opt = helpers.parser()

    # Initialize JAX for multi-process runs
    if opt['multiprocess'] is not None:
        if opt['multiprocess'] == 'bootstrap':
            jax.distributed.initialize()
        else:
            coordinator, num_procs, my_proc = opt['multiprocess'].split(',')
            jax.distributed.initialize(
                coordinator_address=coordinator,
                num_processes=int(num_procs),
                process_id=int(my_proc)
            )

    fft_dims = opt['shape']
    cycles = opt['cycles']

    impl = opt['implementation']
    if impl == "cufftmp":
        dist_fft = cufftmp
    elif impl == "xfft":
        dist_fft = xfft
    else:
        raise ValueError(f"Wrong implementation: got {impl}, expected cufftmp or xfft")

    dist = Dist.create(opt['dist'])
    input_shape = dist.slab_shape(fft_dims)
    dtype = jnp.complex64

    mesh = Mesh(np.asarray(jax.devices()), ('gpus',))
    sharding = jax.sharding.NamedSharding(mesh, dist.part_spec)

    with jax.spmd_mode('allow_all'):

        if opt['mode'] == "test":

            seed = 170
            key = jax.random.PRNGKey(seed)
            input = jax.random.normal(key, shape=fft_dims, dtype=dtype)

            with mesh:

                fft = jax.jit(dist_fft,
                              in_shardings=sharding,
                              out_shardings=sharding,
                              static_argnums=[1, 2])

                output = fft(input, dist, Dir.FWD)

            output_ref = jnp.fft.fftn(input)
            error = jnp.linalg.norm(output - output_ref) / \
                jnp.linalg.norm(output_ref)

            if jax.process_index() == 0:
                print(f"{impl} (test): {fft_dims}, dist {dist} --> {dist.opposite}, num GPUs {jax.device_count()}, num processes {jax.process_count()}, L2 rel error {error:.2e}")
                if error < 1e-4:
                    print("&&&& PASSED")
                else:
                    print("&&&& FAILED")
                    sys.exit(1)

        else:

            with mesh:

                # Performance testing only supports 1 device per process
                # because of the way the input `dinput` is generated
                assert jax.local_device_count() == 1

                # Quick generation of the local array
                input = jnp.ones(input_shape, dtype=dtype)

                # Create the global sharded array
                dinput = jax.make_array_from_single_device_arrays(
                    fft_dims,
                    sharding,
                    [input])

                # Function to benchmark.
                # One forward distributed FFT and one backward distributed FFT.
                def fwd_bwd(x, dist, dir):
                    return dist_fft(dist_fft(x, dist, dir),
                                    dist.opposite,
                                    dir.opposite)

                fwd_bwd_jit = jax.jit(fwd_bwd,
                                      in_shardings=sharding,
                                      out_shardings=sharding,
                                      static_argnums=[1, 2])

                def fwd_bwd_bench(x):
                    return fwd_bwd_jit(x, dist, Dir.FWD)

                # Warmup
                x = fwd_bwd_bench(dinput).block_until_ready()

                # Benchmark
                times = []
                x = dinput
                for _ in range(cycles):
                    cycle_start = time.time()
                    x = fwd_bwd_bench(x)
                    x.block_until_ready()
                    cycle_end = time.time()
                    times.append((cycle_end - cycle_start)/2) # divide by 2 since two FFTs in forward-backward transform
                doutput = x.block_until_ready()

                # Check error
                doutput_ref = dinput
                error = helpers.Frob_error(doutput_ref, doutput, mesh, dist)

                # Calculate statistics, and multiply 1e3 to convert to ms
                mean_time = np.mean(times) * 1e3
                median_time = np.median(times) * 1e3
                std_dev = np.std(times) * 1e3
                min_time = np.min(times) * 1e3
                max_time = np.max(times) * 1e3

                # Perf ?
                perf_GFlops = \
                    (5 * math.prod(fft_dims) * math.log2(math.prod(fft_dims))) / 1e9 / median_time
                bandwidth_GBsGPUdir = \
                    (8 * math.prod(fft_dims)) / jax.device_count()             / 1e9 / median_time

                if jax.process_index() == 0:
                    print(f"{impl} (perf): {fft_dims}, num GPUs {jax.device_count()}, num processes {jax.process_count()}, relative L2 error {error:.2e}, cycles {cycles}, time_avg {mean_time:.2e} ms, time_med {median_time:.2e} ms, time_std {std_dev:.2e} ms, time_min {min_time:.2e} ms, time_max {max_time:.2e} ms, perf_median {perf_GFlops:.2e} GFlop/s, bandwidth_median {bandwidth_GBsGPUdir:.2e} GB/s/GPU")
                    if error < 1e-4:
                        print("&&&& PASSED")
                    else:
                        print("&&&& FAILED")
                        sys.exit(1)

if __name__ == "__main__":
    main()


main

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time
import math
import numpy as np
import sys

import jax
import jax.numpy as jnp
from jax.sharding import Mesh

from fft_common import Dist, Dir
from cufftmp_jax import cufftmp
from xfft import xfft

import helpers

def main():

    opt = helpers.parser()

    # Initialize JAX for multi-process runs
    if opt['multiprocess'] is not None:
        if opt['multiprocess'] == 'bootstrap':
            jax.distributed.initialize()
        else:
            coordinator, num_procs, my_proc = opt['multiprocess'].split(',')
            jax.distributed.initialize(
                coordinator_address=coordinator,
                num_processes=int(num_procs),
                process_id=int(my_proc)
            )

    fft_dims = opt['shape']
    cycles = opt['cycles']

    impl = opt['implementation']
    if impl == "cufftmp":
        dist_fft = cufftmp
    elif impl == "xfft":
        dist_fft = xfft
    else:
        raise ValueError(f"Wrong implementation: got {impl}, expected cufftmp or xfft")

    dist = Dist.create(opt['dist'])
    input_shape = dist.slab_shape(fft_dims)
    dtype = jnp.complex64

    mesh = Mesh(np.asarray(jax.devices()), ('gpus',))
    sharding = jax.sharding.NamedSharding(mesh, dist.part_spec)

    with jax.spmd_mode('allow_all'):

        if opt['mode'] == "test":

            seed = 170
            key = jax.random.PRNGKey(seed)
            input = jax.random.normal(key, shape=fft_dims, dtype=dtype)

            with mesh:

                fft = jax.jit(dist_fft,
                              in_shardings=sharding,
                              out_shardings=sharding,
                              static_argnums=[1, 2])

                output = fft(input, dist, Dir.FWD)

            output_ref = jnp.fft.fftn(input)
            error = jnp.linalg.norm(output - output_ref) / \
                jnp.linalg.norm(output_ref)

            if jax.process_index() == 0:
                print(f"{impl} (test): {fft_dims}, dist {dist} --> {dist.opposite}, num GPUs {jax.device_count()}, num processes {jax.process_count()}, L2 rel error {error:.2e}")
                if error < 1e-4:
                    print("&&&& PASSED")
                else:
                    print("&&&& FAILED")
                    sys.exit(1)

        else:

            with mesh:

                # Performance testing only supports 1 device per process
                # because of the way the input `dinput` is generated
                assert jax.local_device_count() == 1

                # Quick generation of the local array
                input = jnp.ones(input_shape, dtype=dtype)

                # Create the global sharded array
                dinput = jax.make_array_from_single_device_arrays(
                    fft_dims,
                    sharding,
                    [input])

                # Function to benchmark.
                # One forward distributed FFT and one backward distributed FFT.
                def fwd_bwd(x, dist, dir):
                    return dist_fft(dist_fft(x, dist, dir),
                                    dist.opposite,
                                    dir.opposite)

                fwd_bwd_jit = jax.jit(fwd_bwd,
                                      in_shardings=sharding,
                                      out_shardings=sharding,
                                      static_argnums=[1, 2])

                def fwd_bwd_bench(x):
                    return fwd_bwd_jit(x, dist, Dir.FWD)

                # Warmup
                x = fwd_bwd_bench(dinput).block_until_ready()

                # Benchmark
                times = []
                x = dinput
                for _ in range(cycles):
                    cycle_start = time.time()
                    x = fwd_bwd_bench(x)
                    x.block_until_ready()
                    cycle_end = time.time()
                    times.append((cycle_end - cycle_start)/2) # divide by 2 since two FFTs in forward-backward transform
                doutput = x.block_until_ready()

                # Check error
                doutput_ref = dinput
                error = helpers.Frob_error(doutput_ref, doutput, mesh, dist)

                # Calculate statistics, and multiply 1e3 to convert to ms
                mean_time = np.mean(times) * 1e3
                median_time = np.median(times) * 1e3
                std_dev = np.std(times) * 1e3
                min_time = np.min(times) * 1e3
                max_time = np.max(times) * 1e3

                # Perf ?
                perf_GFlops = \
                    (5 * math.prod(fft_dims) * math.log2(math.prod(fft_dims))) / 1e9 / median_time
                bandwidth_GBsGPUdir = \
                    (8 * math.prod(fft_dims)) / jax.device_count()             / 1e9 / median_time

                if jax.process_index() == 0:
                    print(f"{impl} (perf): {fft_dims}, num GPUs {jax.device_count()}, num processes {jax.process_count()}, relative L2 error {error:.2e}, cycles {cycles}, time_avg {mean_time:.2e} ms, time_med {median_time:.2e} ms, time_std {std_dev:.2e} ms, time_min {min_time:.2e} ms, time_max {max_time:.2e} ms, perf_median {perf_GFlops:.2e} GFlop/s, bandwidth_median {bandwidth_GBsGPUdir:.2e} GB/s/GPU")
                    if error < 1e-4:
                        print("&&&& PASSED")
                    else:
                        print("&&&& FAILED")
                        sys.exit(1)

if __name__ == "__main__":
    main()


        else:

            with mesh:

                # Performance testing only supports 1 device per process
                # because of the way the input `dinput` is generated
                assert jax.local_device_count() == 1

                # Quick generation of the local array
                input = jnp.ones(input_shape, dtype=dtype)

                # Create the global sharded array
                dinput = jax.make_array_from_single_device_arrays(
                    fft_dims,
                    sharding,
                    [input])

                # Function to benchmark.
                # One forward distributed FFT and one backward distributed FFT.
                def fwd_bwd(x, dist, dir):
                    return dist_fft(dist_fft(x, dist, dir),
                                    dist.opposite,
                                    dir.opposite)

                fwd_bwd_jit = jax.jit(fwd_bwd,
                                      in_shardings=sharding,
                                      out_shardings=sharding,
                                      static_argnums=[1, 2])

                def fwd_bwd_bench(x):
                    return fwd_bwd_jit(x, dist, Dir.FWD)

                # Warmup
                x = fwd_bwd_bench(dinput).block_until_ready()

                # Benchmark
                times = []
                x = dinput
                for _ in range(cycles):
                    cycle_start = time.time()
                    x = fwd_bwd_bench(x)
                    x.block_until_ready()
                    cycle_end = time.time()


                input = jnp.ones(input_shape, dtype=dtype)

                # Create the global sharded array
                dinput = jax.make_array_from_single_device_arrays(
                    fft_dims,
                    sharding,
                    [input])

                # Function to benchmark.
                # One forward distributed FFT and one backward distributed FFT.
                def fwd_bwd(x, dist, dir):
                    return dist_fft(dist_fft(x, dist, dir),
                                    dist.opposite,
                                    dir.opposite)

                fwd_bwd_jit = jax.jit(fwd_bwd,
                                      in_shardings=sharding,
                                      out_shardings=sharding,
                                      static_argnums=[1, 2])

                def fwd_bwd_bench(x):
                    return fwd_bwd_jit(x, dist, Dir.FWD)

                # Warmup
                x = fwd_bwd_bench(dinput).block_until_ready()

                # Benchmark
                times = []
                x = dinput
                for _ in range(cycles):
                    cycle_start = time.time()
                    x = fwd_bwd_bench(x)
                    x.block_until_ready()
                    cycle_end = time.time()
                    times.append((cycle_end - cycle_start)/2) # divide by 2 since two FFTs in forward-backward transform
                doutput = x.block_until_ready()

                # Check error
                doutput_ref = dinput
                error = helpers.Frob_error(doutput_ref, doutput, mesh, dist)

                # Calculate statistics, and multiply 1e3 to convert to ms


# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from enum import Enum

import jax
from jax.sharding import PartitionSpec

class Dist(Enum):

    """Describes a SLAB data decomposition

    For a X*Y*Z array, SLABS_X indicates the array is
    distributed along the first dimension, i.e., each
    device owns a slab of size (X // nGPUs)*Y*Z
    SLABS_Y indicates the array is distributed along the
    second dimension, with each device owning a slab
    of size X*(Y // nGPUs)*Z.
    """

    SLABS_X = 'SLABS_X'
    SLABS_Y = 'SLABS_Y'

    @staticmethod
    def create(string):
        if string == 'X':
            return Dist.SLABS_X
        elif string == 'Y':
            return Dist.SLABS_Y
        else:
            raise RuntimeError("Wrong dist")

    @property
    def opposite(dist):
        if dist == Dist.SLABS_X:
            return Dist.SLABS_Y
        else:
            return Dist.SLABS_X

    @property
    def _C_enum(dist):
        if dist == Dist.SLABS_X:
            return 0
        else:
            return 1

    def fft_axes(self, fft_rank):
        if self == Dist.SLABS_X:
            return list(range(1, fft_rank))
        else:
            return [0]

    def slab_shape(self, fft_dims):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (
                fft_dims[0] // ngpus,
                fft_dims[1],
                *fft_dims[2:]
            )
        else:
            return (
                fft_dims[0],
                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])
        else:
            return (local_shape[0], local_shape[1] * ngpus, *local_shape[2:])

    @property
    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'
    INV = 'INV'

    @property
    def _C_enum(dir):
        if dir == Dir.FWD:
            return 0
        else:
            return 1

    @property
    def opposite(dir):
        if dir == Dir.FWD:
            return Dir.INV
        else:
            return Dir.FWD


Dist

# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from enum import Enum

import jax
from jax.sharding import PartitionSpec

class Dist(Enum):

    """Describes a SLAB data decomposition

    For a X*Y*Z array, SLABS_X indicates the array is
    distributed along the first dimension, i.e., each
    device owns a slab of size (X // nGPUs)*Y*Z
    SLABS_Y indicates the array is distributed along the
    second dimension, with each device owning a slab
    of size X*(Y // nGPUs)*Z.
    """

    SLABS_X = 'SLABS_X'
    SLABS_Y = 'SLABS_Y'

    @staticmethod
    def create(string):
        if string == 'X':
            return Dist.SLABS_X
        elif string == 'Y':
            return Dist.SLABS_Y
        else:
            raise RuntimeError("Wrong dist")

    @property
    def opposite(dist):
        if dist == Dist.SLABS_X:
            return Dist.SLABS_Y
        else:
            return Dist.SLABS_X

    @property
    def _C_enum(dist):
        if dist == Dist.SLABS_X:
            return 0
        else:
            return 1

    def fft_axes(self, fft_rank):
        if self == Dist.SLABS_X:
            return list(range(1, fft_rank))
        else:
            return [0]

    def slab_shape(self, fft_dims):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (
                fft_dims[0] // ngpus,
                fft_dims[1],
                *fft_dims[2:]
            )
        else:
            return (
                fft_dims[0],
                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])
        else:
            return (local_shape[0], local_shape[1] * ngpus, *local_shape[2:])

    @property
    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'
    INV = 'INV'

    @property
    def _C_enum(dir):
        if dir == Dir.FWD:
            return 0
        else:
            return 1


Dir

                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])
        else:
            return (local_shape[0], local_shape[1] * ngpus, *local_shape[2:])

    @property
    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'
    INV = 'INV'

    @property
    def _C_enum(dir):
        if dir == Dir.FWD:
            return 0
        else:
            return 1

    @property
    def opposite(dir):
        if dir == Dir.FWD:
            return Dir.INV
        else:
            return Dir.FWD


create

import jax
from jax.sharding import PartitionSpec

class Dist(Enum):

    """Describes a SLAB data decomposition

    For a X*Y*Z array, SLABS_X indicates the array is
    distributed along the first dimension, i.e., each
    device owns a slab of size (X // nGPUs)*Y*Z
    SLABS_Y indicates the array is distributed along the
    second dimension, with each device owning a slab
    of size X*(Y // nGPUs)*Z.
    """

    SLABS_X = 'SLABS_X'
    SLABS_Y = 'SLABS_Y'

    @staticmethod
    def create(string):
        if string == 'X':
            return Dist.SLABS_X
        elif string == 'Y':
            return Dist.SLABS_Y
        else:
            raise RuntimeError("Wrong dist")

    @property
    def opposite(dist):
        if dist == Dist.SLABS_X:
            return Dist.SLABS_Y
        else:
            return Dist.SLABS_X

    @property
    def _C_enum(dist):
        if dist == Dist.SLABS_X:
            return 0
        else:
            return 1

    def fft_axes(self, fft_rank):
        if self == Dist.SLABS_X:
            return list(range(1, fft_rank))
        else:
            return [0]


opposite

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'
    INV = 'INV'

    @property
    def _C_enum(dir):
        if dir == Dir.FWD:
            return 0
        else:
            return 1

    @property
    def opposite(dir):
        if dir == Dir.FWD:
            return Dir.INV
        else:
            return Dir.FWD


    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'
    INV = 'INV'

    @property
    def _C_enum(dir):
        if dir == Dir.FWD:
            return 0
        else:
            return 1

    @property
    def opposite(dir):
        if dir == Dir.FWD:
            return Dir.INV
        else:
            return Dir.FWD


            return Dist.SLABS_X
        elif string == 'Y':
            return Dist.SLABS_Y
        else:
            raise RuntimeError("Wrong dist")

    @property
    def opposite(dist):
        if dist == Dist.SLABS_X:
            return Dist.SLABS_Y
        else:
            return Dist.SLABS_X

    @property
    def _C_enum(dist):
        if dist == Dist.SLABS_X:
            return 0
        else:
            return 1

    def fft_axes(self, fft_rank):
        if self == Dist.SLABS_X:
            return list(range(1, fft_rank))
        else:
            return [0]

    def slab_shape(self, fft_dims):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (
                fft_dims[0] // ngpus,
                fft_dims[1],
                *fft_dims[2:]
            )
        else:
            return (
                fft_dims[0],
                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])


    @property
    def opposite(dist):
        if dist == Dist.SLABS_X:
            return Dist.SLABS_Y
        else:
            return Dist.SLABS_X

    @property
    def _C_enum(dist):
        if dist == Dist.SLABS_X:
            return 0
        else:
            return 1

    def fft_axes(self, fft_rank):
        if self == Dist.SLABS_X:
            return list(range(1, fft_rank))
        else:
            return [0]

    def slab_shape(self, fft_dims):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (
                fft_dims[0] // ngpus,
                fft_dims[1],
                *fft_dims[2:]
            )
        else:
            return (
                fft_dims[0],
                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])
        else:
            return (local_shape[0], local_shape[1] * ngpus, *local_shape[2:])

    @property
    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction


        if self == Dist.SLABS_X:
            return list(range(1, fft_rank))
        else:
            return [0]

    def slab_shape(self, fft_dims):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (
                fft_dims[0] // ngpus,
                fft_dims[1],
                *fft_dims[2:]
            )
        else:
            return (
                fft_dims[0],
                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])
        else:
            return (local_shape[0], local_shape[1] * ngpus, *local_shape[2:])

    @property
    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'


                fft_dims[0] // ngpus,
                fft_dims[1],
                *fft_dims[2:]
            )
        else:
            return (
                fft_dims[0],
                fft_dims[1] // ngpus,
                *fft_dims[2:]
            )

    def fft_shape(self, local_shape):
        ngpus = jax.device_count()
        if self == Dist.SLABS_X:
            return (local_shape[0] * ngpus, local_shape[1], *local_shape[2:])
        else:
            return (local_shape[0], local_shape[1] * ngpus, *local_shape[2:])

    @property
    def part_spec(dist):
        if dist == Dist.SLABS_X:
            return PartitionSpec("gpus", None)
        else:
            return PartitionSpec(None, "gpus")

class Dir(Enum):

    """Describe the FFT direction

    FWD is the forward, unnormalized, direction.
    BWD is the backward, normalized by 1/N, direction,
    with N the product of the dimensions.
    """

    FWD = 'FWD'
    INV = 'INV'

    @property
    def _C_enum(dir):
        if dir == Dir.FWD:
            return 0
        else:
            return 1


# -*- coding: utf-8 -*-

# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__all__ = ["cufftmp"]

from functools import partial
import math

import jax
from jax.lib import xla_client
from jax import core, dtypes
from jax.interpreters import xla, mlir
from jax.core import ShapedArray
from jax.sharding import NamedSharding
from jax.experimental.custom_partitioning import custom_partitioning
from jaxlib.hlo_helpers import custom_call

from fft_common import Dir, Dist

from . import gpu_ops
for _name, _value in gpu_ops.registrations().items():
    xla_client.register_custom_call_target(_name, _value, platform="gpu")

xops = xla_client.ops

# ************
# * BINDINGS *
# ************

def _cufftmp_bind(input, num_parts, dist, dir):

    # param=val means it's a static parameter
    (output,) = _cufftmp_prim.bind(input,
                                   num_parts=num_parts,
                                   dist=dist,
                                   dir=dir)

    # scale in INVERSE direction
    if dir == Dir.INV:
        fft_dims = dist.fft_shape(input.shape)
        output = jax.numpy.divide(output, math.prod([
            jax.numpy.complex64(f) for f in fft_dims
        ]))

    return output

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    """ Describes the required input and output sharding of the op.
    `mesh`, `arg_shapes` and `result_shape` are the shapes provided by the
    user (i.e., in jit).
    Returns:
    - The operation to perform locally on all GPUs
    - The output sharding
    - The input sharding """

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)

    return mesh, \
        lambda x: _cufftmp_bind(x,
                                num_parts=jax.device_count(),
                                dist=dist,
                                dir=dir), \
        _supported_sharding(arg_shardings[0], dist.opposite), \
        (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def cufftmp(x, dist, dir):

    """Compute the DFT using a JAX+cuFFTMp implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(
            cufftmp,
            in_shardings=sharding,
            out_shardings=sharding,
            static_argnums=[1, 2]
            )(x, dist, dir)

    """

    # cuFFTMp only supports 1 device per proces
    assert jax.local_device_count() == 1

    @custom_partitioning
    def _cufftmp_(x):
        return _cufftmp_bind(x, num_parts=1, dist=dist, dir=dir)

    _cufftmp_.def_partition(
        infer_sharding_from_operands=partial(
            _infer_sharding_from_operands,
            dist=dist,
            dir=dir),
        partition=partial(
            _partition,
            dist=dist,
            dir=dir))

    return _cufftmp_(x)

# *********************************
# *  SUPPORT FOR JIT COMPILATION  *
# *********************************

# Abstract implementation, i.e., return the shape of the output array
# based on the input array and a number of partitions (ie devices)
def _cufftmp_abstract(input, num_parts, dist, dir):
    dtype = dtypes.canonicalize_dtype(input.dtype)
    input_shape = input.shape
    if dist == Dist.SLABS_X:
        output_shape = (input_shape[0] * num_parts,
                        input_shape[1] // num_parts,
                        *input_shape[2:])
    elif dist == Dist.SLABS_Y:
        output_shape = (input_shape[0] // num_parts,
                        input_shape[1] * num_parts,
                        *input_shape[2:])
    return (ShapedArray(output_shape, dtype),)

# Implementation calling into the C++ bindings
def _cufftmp_lowering(ctx, input, num_parts, dist, dir):

    assert num_parts == jax.device_count()

    input_type = mlir.ir.RankedTensorType(input.type)
    dims_in = input_type.shape
    fft_dims = dist.fft_shape(dims_in)
    dims_out = dist.opposite.slab_shape(fft_dims)
    output_type = mlir.ir.RankedTensorType.get(
        dims_out,
        input_type.element_type
    )

    layout = tuple(range(len(dims_in) - 1, -1, -1))

    if len(fft_dims) == 2:
        opaque = gpu_ops.build_cufftmp_descriptor(
            fft_dims[0],
            fft_dims[1],
            1,
            dist._C_enum,
            dir._C_enum
        )
    elif len(fft_dims) == 3:
        opaque = gpu_ops.build_cufftmp_descriptor(
            fft_dims[0],
            fft_dims[1],
            fft_dims[2],
            dist._C_enum,
            dir._C_enum
        )
    else:
        raise ValueError("Unsupported tensor rank; must be 2 or 3")

    return custom_call(
        "gpu_cufftmp",
        # Output types
        result_types=[output_type],
        # The inputs:
        operands=[input,],
        # Layout specification:
        operand_layouts=[layout,],
        result_layouts=[layout,],
        # GPU specific additional data
        backend_config=opaque
    ).results

# *********************************************
# *  BOILERPLATE TO REGISTER THE OP WITH JAX  *
# *********************************************
_cufftmp_prim = core.Primitive("cufftmp")
_cufftmp_prim.multiple_results = True
_cufftmp_prim.def_impl(partial(xla.apply_primitive, _cufftmp_prim))
_cufftmp_prim.def_abstract_eval(_cufftmp_abstract)

# Register the op with MLIR
mlir.register_lowering(_cufftmp_prim, _cufftmp_lowering, platform="gpu")


from jax import core, dtypes
from jax.interpreters import xla, mlir
from jax.core import ShapedArray
from jax.sharding import NamedSharding
from jax.experimental.custom_partitioning import custom_partitioning
from jaxlib.hlo_helpers import custom_call

from fft_common import Dir, Dist

from . import gpu_ops
for _name, _value in gpu_ops.registrations().items():
    xla_client.register_custom_call_target(_name, _value, platform="gpu")

xops = xla_client.ops

# ************
# * BINDINGS *
# ************

def _cufftmp_bind(input, num_parts, dist, dir):

    # param=val means it's a static parameter
    (output,) = _cufftmp_prim.bind(input,
                                   num_parts=num_parts,
                                   dist=dist,
                                   dir=dir)

    # scale in INVERSE direction
    if dir == Dir.INV:
        fft_dims = dist.fft_shape(input.shape)
        output = jax.numpy.divide(output, math.prod([
            jax.numpy.complex64(f) for f in fft_dims
        ]))

    return output

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    """ Describes the required input and output sharding of the op.
    `mesh`, `arg_shapes` and `result_shape` are the shapes provided by the
    user (i.e., in jit).
    Returns:
    - The operation to perform locally on all GPUs
    - The output sharding
    - The input sharding """


def _cufftmp_bind(input, num_parts, dist, dir):

    # param=val means it's a static parameter
    (output,) = _cufftmp_prim.bind(input,
                                   num_parts=num_parts,
                                   dist=dist,
                                   dir=dir)

    # scale in INVERSE direction
    if dir == Dir.INV:
        fft_dims = dist.fft_shape(input.shape)
        output = jax.numpy.divide(output, math.prod([
            jax.numpy.complex64(f) for f in fft_dims
        ]))

    return output

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    """ Describes the required input and output sharding of the op.
    `mesh`, `arg_shapes` and `result_shape` are the shapes provided by the
    user (i.e., in jit).
    Returns:
    - The operation to perform locally on all GPUs
    - The output sharding
    - The input sharding """

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)

    return mesh, \
        lambda x: _cufftmp_bind(x,


    # param=val means it's a static parameter
    (output,) = _cufftmp_prim.bind(input,
                                   num_parts=num_parts,
                                   dist=dist,
                                   dir=dir)

    # scale in INVERSE direction
    if dir == Dir.INV:
        fft_dims = dist.fft_shape(input.shape)
        output = jax.numpy.divide(output, math.prod([
            jax.numpy.complex64(f) for f in fft_dims
        ]))

    return output

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    """ Describes the required input and output sharding of the op.
    `mesh`, `arg_shapes` and `result_shape` are the shapes provided by the
    user (i.e., in jit).
    Returns:
    - The operation to perform locally on all GPUs
    - The output sharding
    - The input sharding """

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)

    return mesh, \
        lambda x: _cufftmp_bind(x,
                                num_parts=jax.device_count(),
                                dist=dist,
                                dir=dir), \
        _supported_sharding(arg_shardings[0], dist.opposite), \
        (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def cufftmp(x, dist, dir):

    """Compute the DFT using a JAX+cuFFTMp implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist


    """ Describes the required input and output sharding of the op.
    `mesh`, `arg_shapes` and `result_shape` are the shapes provided by the
    user (i.e., in jit).
    Returns:
    - The operation to perform locally on all GPUs
    - The output sharding
    - The input sharding """

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)

    return mesh, \
        lambda x: _cufftmp_bind(x,
                                num_parts=jax.device_count(),
                                dist=dist,
                                dir=dir), \
        _supported_sharding(arg_shardings[0], dist.opposite), \
        (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def cufftmp(x, dist, dir):

    """Compute the DFT using a JAX+cuFFTMp implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(
            cufftmp,


cufftmp

    return mesh, \
        lambda x: _cufftmp_bind(x,
                                num_parts=jax.device_count(),
                                dist=dist,
                                dir=dir), \
        _supported_sharding(arg_shardings[0], dist.opposite), \
        (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def cufftmp(x, dist, dir):

    """Compute the DFT using a JAX+cuFFTMp implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(
            cufftmp,
            in_shardings=sharding,
            out_shardings=sharding,
            static_argnums=[1, 2]
            )(x, dist, dir)

    """

    # cuFFTMp only supports 1 device per proces
    assert jax.local_device_count() == 1

    @custom_partitioning
    def _cufftmp_(x):
        return _cufftmp_bind(x, num_parts=1, dist=dist, dir=dir)

    _cufftmp_.def_partition(
        infer_sharding_from_operands=partial(
            _infer_sharding_from_operands,
            dist=dist,
            dir=dir),
        partition=partial(
            _partition,
            dist=dist,
            dir=dir))

    return _cufftmp_(x)

# *********************************
# *  SUPPORT FOR JIT COMPILATION  *
# *********************************

# Abstract implementation, i.e., return the shape of the output array
# based on the input array and a number of partitions (ie devices)
def _cufftmp_abstract(input, num_parts, dist, dir):
    dtype = dtypes.canonicalize_dtype(input.dtype)
    input_shape = input.shape
    if dist == Dist.SLABS_X:
        output_shape = (input_shape[0] * num_parts,
                        input_shape[1] // num_parts,
                        *input_shape[2:])
    elif dist == Dist.SLABS_Y:
        output_shape = (input_shape[0] // num_parts,
                        input_shape[1] * num_parts,
                        *input_shape[2:])
    return (ShapedArray(output_shape, dtype),)


    _cufftmp_.def_partition(
        infer_sharding_from_operands=partial(
            _infer_sharding_from_operands,
            dist=dist,
            dir=dir),
        partition=partial(
            _partition,
            dist=dist,
            dir=dir))

    return _cufftmp_(x)

# *********************************
# *  SUPPORT FOR JIT COMPILATION  *
# *********************************

# Abstract implementation, i.e., return the shape of the output array
# based on the input array and a number of partitions (ie devices)
def _cufftmp_abstract(input, num_parts, dist, dir):
    dtype = dtypes.canonicalize_dtype(input.dtype)
    input_shape = input.shape
    if dist == Dist.SLABS_X:
        output_shape = (input_shape[0] * num_parts,
                        input_shape[1] // num_parts,
                        *input_shape[2:])
    elif dist == Dist.SLABS_Y:
        output_shape = (input_shape[0] // num_parts,
                        input_shape[1] * num_parts,
                        *input_shape[2:])
    return (ShapedArray(output_shape, dtype),)

# Implementation calling into the C++ bindings
def _cufftmp_lowering(ctx, input, num_parts, dist, dir):

    assert num_parts == jax.device_count()

    input_type = mlir.ir.RankedTensorType(input.type)
    dims_in = input_type.shape
    fft_dims = dist.fft_shape(dims_in)
    dims_out = dist.opposite.slab_shape(fft_dims)
    output_type = mlir.ir.RankedTensorType.get(
        dims_out,
        input_type.element_type
    )

    layout = tuple(range(len(dims_in) - 1, -1, -1))

    if len(fft_dims) == 2:
        opaque = gpu_ops.build_cufftmp_descriptor(


# *  SUPPORT FOR JIT COMPILATION  *
# *********************************

# Abstract implementation, i.e., return the shape of the output array
# based on the input array and a number of partitions (ie devices)
def _cufftmp_abstract(input, num_parts, dist, dir):
    dtype = dtypes.canonicalize_dtype(input.dtype)
    input_shape = input.shape
    if dist == Dist.SLABS_X:
        output_shape = (input_shape[0] * num_parts,
                        input_shape[1] // num_parts,
                        *input_shape[2:])
    elif dist == Dist.SLABS_Y:
        output_shape = (input_shape[0] // num_parts,
                        input_shape[1] * num_parts,
                        *input_shape[2:])
    return (ShapedArray(output_shape, dtype),)

# Implementation calling into the C++ bindings
def _cufftmp_lowering(ctx, input, num_parts, dist, dir):

    assert num_parts == jax.device_count()

    input_type = mlir.ir.RankedTensorType(input.type)
    dims_in = input_type.shape
    fft_dims = dist.fft_shape(dims_in)
    dims_out = dist.opposite.slab_shape(fft_dims)
    output_type = mlir.ir.RankedTensorType.get(
        dims_out,
        input_type.element_type
    )

    layout = tuple(range(len(dims_in) - 1, -1, -1))

    if len(fft_dims) == 2:
        opaque = gpu_ops.build_cufftmp_descriptor(
            fft_dims[0],
            fft_dims[1],
            1,
            dist._C_enum,
            dir._C_enum
        )
    elif len(fft_dims) == 3:
        opaque = gpu_ops.build_cufftmp_descriptor(
            fft_dims[0],
            fft_dims[1],
            fft_dims[2],
            dist._C_enum,
            dir._C_enum
        )
    else:
        raise ValueError("Unsupported tensor rank; must be 2 or 3")

    return custom_call(
        "gpu_cufftmp",
        # Output types
        result_types=[output_type],
        # The inputs:
        operands=[input,],
        # Layout specification:
        operand_layouts=[layout,],
        result_layouts=[layout,],
        # GPU specific additional data
        backend_config=opaque
    ).results

# *********************************************
# *  BOILERPLATE TO REGISTER THE OP WITH JAX  *
# *********************************************
_cufftmp_prim = core.Primitive("cufftmp")
_cufftmp_prim.multiple_results = True
_cufftmp_prim.def_impl(partial(xla.apply_primitive, _cufftmp_prim))
_cufftmp_prim.def_abstract_eval(_cufftmp_abstract)

# Register the op with MLIR
mlir.register_lowering(_cufftmp_prim, _cufftmp_lowering, platform="gpu")


            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(
            cufftmp,
            in_shardings=sharding,
            out_shardings=sharding,
            static_argnums=[1, 2]
            )(x, dist, dir)

    """

    # cuFFTMp only supports 1 device per proces
    assert jax.local_device_count() == 1

    @custom_partitioning
    def _cufftmp_(x):
        return _cufftmp_bind(x, num_parts=1, dist=dist, dir=dir)

    _cufftmp_.def_partition(
        infer_sharding_from_operands=partial(
            _infer_sharding_from_operands,
            dist=dist,
            dir=dir),
        partition=partial(
            _partition,
            dist=dist,
            dir=dir))

    return _cufftmp_(x)

# *********************************
# *  SUPPORT FOR JIT COMPILATION  *
# *********************************

# Abstract implementation, i.e., return the shape of the output array
# based on the input array and a number of partitions (ie devices)


# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import partial

import jax
from jax.sharding import NamedSharding
from jax.experimental.custom_partitioning import custom_partitioning
from fft_common import Dir

def _fft(x, dist, dir):

    """ Compute a local FFT along the appropriate axes (based on dist), in the
    forward or backward direction """

    if dir == Dir.FWD:
        return jax.numpy.fft.fftn(x, axes=dist.fft_axes(len(x.shape)))
    else:
        return jax.numpy.fft.ifftn(x, axes=dist.fft_axes(len(x.shape)))

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):
    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def fft(x, dist, dir):

    """ Extends jax.numpy.fft.fftn to support sharding along the first or
    second direction, without intermediate re-sharding """

    @custom_partitioning
    def _fft_(x):
        return _fft(x, dist, dir)

    _fft_.def_partition(
        infer_sharding_from_operands=partial(_infer_sharding_from_operands,
                                             dist=dist,
                                             dir=dir),
        partition=partial(_partition, dist=dist, dir=dir))

    return _fft_(x)

def xfft(x, dist, dir):

    """Compute the discrete Fourier transform using a JAX-only implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(xfft,
            in_shardings=sharding,
            out_shardings=sharding_opposite,
            static_argnums=[1, 2]
            )(x, dist, dir)
    """

    # If dist == Dist.SLABS_X, FFT along Y and Z
    x = fft(x, dist, dir)

    # Implicitly re-shards to match the required
    # input sharding of the next fft(..., dist.opposite, ...)

    # If dist == Dist.SLABS_X, FFT along X
    x = fft(x, dist.opposite, dir)

    return x


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import partial

import jax
from jax.sharding import NamedSharding
from jax.experimental.custom_partitioning import custom_partitioning
from fft_common import Dir

def _fft(x, dist, dir):

    """ Compute a local FFT along the appropriate axes (based on dist), in the
    forward or backward direction """

    if dir == Dir.FWD:
        return jax.numpy.fft.fftn(x, axes=dist.fft_axes(len(x.shape)))
    else:
        return jax.numpy.fft.ifftn(x, axes=dist.fft_axes(len(x.shape)))

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,


from functools import partial

import jax
from jax.sharding import NamedSharding
from jax.experimental.custom_partitioning import custom_partitioning
from fft_common import Dir

def _fft(x, dist, dir):

    """ Compute a local FFT along the appropriate axes (based on dist), in the
    forward or backward direction """

    if dir == Dir.FWD:
        return jax.numpy.fft.fftn(x, axes=dist.fft_axes(len(x.shape)))
    else:
        return jax.numpy.fft.ifftn(x, axes=dist.fft_axes(len(x.shape)))

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):
    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)


from jax.sharding import NamedSharding
from jax.experimental.custom_partitioning import custom_partitioning
from fft_common import Dir

def _fft(x, dist, dir):

    """ Compute a local FFT along the appropriate axes (based on dist), in the
    forward or backward direction """

    if dir == Dir.FWD:
        return jax.numpy.fft.fftn(x, axes=dist.fft_axes(len(x.shape)))
    else:
        return jax.numpy.fft.ifftn(x, axes=dist.fft_axes(len(x.shape)))

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):
    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def fft(x, dist, dir):

    """ Extends jax.numpy.fft.fftn to support sharding along the first or
    second direction, without intermediate re-sharding """

    @custom_partitioning
    def _fft_(x):
        return _fft(x, dist, dir)


    else:
        return jax.numpy.fft.ifftn(x, axes=dist.fft_axes(len(x.shape)))

def _supported_sharding(sharding, dist):
    return NamedSharding(sharding.mesh, dist.part_spec)

def _partition(mesh,
               arg_shapes,
               result_shape,
               dist,
               dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):
    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def fft(x, dist, dir):

    """ Extends jax.numpy.fft.fftn to support sharding along the first or
    second direction, without intermediate re-sharding """

    @custom_partitioning
    def _fft_(x):
        return _fft(x, dist, dir)

    _fft_.def_partition(
        infer_sharding_from_operands=partial(_infer_sharding_from_operands,
                                             dist=dist,
                                             dir=dir),
        partition=partial(_partition, dist=dist, dir=dir))

    return _fft_(x)


fft

               arg_shapes,
               result_shape,
               dist,
               dir):

    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):
    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def fft(x, dist, dir):

    """ Extends jax.numpy.fft.fftn to support sharding along the first or
    second direction, without intermediate re-sharding """

    @custom_partitioning
    def _fft_(x):
        return _fft(x, dist, dir)

    _fft_.def_partition(
        infer_sharding_from_operands=partial(_infer_sharding_from_operands,
                                             dist=dist,
                                             dir=dir),
        partition=partial(_partition, dist=dist, dir=dir))

    return _fft_(x)

def xfft(x, dist, dir):

    """Compute the discrete Fourier transform using a JAX-only implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(xfft,
            in_shardings=sharding,


xfft

def fft(x, dist, dir):

    """ Extends jax.numpy.fft.fftn to support sharding along the first or
    second direction, without intermediate re-sharding """

    @custom_partitioning
    def _fft_(x):
        return _fft(x, dist, dir)

    _fft_.def_partition(
        infer_sharding_from_operands=partial(_infer_sharding_from_operands,
                                             dist=dist,
                                             dir=dir),
        partition=partial(_partition, dist=dist, dir=dir))

    return _fft_(x)

def xfft(x, dist, dir):

    """Compute the discrete Fourier transform using a JAX-only implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir

    Returns the transformed tensor.
    The output tensoris distributed according to dist.opposite

    This function should be used with jit like

        jit(xfft,
            in_shardings=sharding,
            out_shardings=sharding_opposite,
            static_argnums=[1, 2]
            )(x, dist, dir)
    """

    # If dist == Dist.SLABS_X, FFT along Y and Z
    x = fft(x, dist, dir)

    # Implicitly re-shards to match the required
    # input sharding of the next fft(..., dist.opposite, ...)

    # If dist == Dist.SLABS_X, FFT along X
    x = fft(x, dist.opposite, dir)

    return x


    return mesh, lambda x: _fft(x, dist, dir), \
           _supported_sharding(arg_shardings[0], dist), \
           (_supported_sharding(arg_shardings[0], dist),)

def _infer_sharding_from_operands(mesh,
                                  arg_shapes,
                                  result_shape,
                                  dist,
                                  dir):
    arg_shardings = jax.tree.map(lambda x: x.sharding, arg_shapes)
    return _supported_sharding(arg_shardings[0], dist)

def fft(x, dist, dir):

    """ Extends jax.numpy.fft.fftn to support sharding along the first or
    second direction, without intermediate re-sharding """

    @custom_partitioning
    def _fft_(x):
        return _fft(x, dist, dir)

    _fft_.def_partition(
        infer_sharding_from_operands=partial(_infer_sharding_from_operands,
                                             dist=dist,
                                             dir=dir),
        partition=partial(_partition, dist=dist, dir=dir))

    return _fft_(x)

def xfft(x, dist, dir):

    """Compute the discrete Fourier transform using a JAX-only implementation.

    Arguments:
    x    -- the input tensor
    dist -- the data decomposition of x.
            Should be an instance of fft_common.Dist
    dir  -- the direction of the transform.
            Should be an instance of fft_common.Dir