/* Generated by Cython 3.2.5 */

/* BEGIN: Cython Metadata
{
    "distutils": {
        "define_macros": [
            [
                "_FORCE_INLINES",
                "1"
            ],
            [
                "CYTHON_EXTERN_C",
                "extern \"C\""
            ],
            [
                "NPY_TARGET_VERSION",
                "NPY_2_0_API_VERSION"
            ],
            [
                "CUPY_CACHE_KEY",
                "28a6d0d64fe089c15e4ff802c7b0f8edb50d2a99"
            ],
            [
                "CUPY_CUB_VERSION_CODE",
                300102
            ],
            [
                "CUPY_JITIFY_VERSION_CODE",
                "1a0ca0e"
            ]
        ],
        "depends": [],
        "extra_compile_args": [
            "-std=c++17"
        ],
        "extra_link_args": [
            "-Wl,--disable-new-dtags,-rpath,/opt/cuda/lib64,-rpath,/opt/cuda/lib"
        ],
        "extra_objects": [
            "/opt/cuda/lib64/libcudart_static.a"
        ],
        "include_dirs": [
            "/build/python-cupy/src/python-cupy/cupy/_core/include/cupy/_cccl/libcudacxx",
            "/build/python-cupy/src/python-cupy/cupy/_core/include/cupy/_cccl/thrust",
            "/build/python-cupy/src/python-cupy/cupy/_core/include/cupy/_cccl/cub",
            "/build/python-cupy/src/python-cupy/cupy/_core/include",
            "/opt/cuda/include",
            "/usr/lib/python3.14/site-packages/numpy/_core/include"
        ],
        "language": "c++",
        "libraries": [
            "pthread",
            "rt",
            "dl",
            "cublas",
            "cufft",
            "curand",
            "cusparse"
        ],
        "library_dirs": [
            "/opt/cuda/lib64",
            "/opt/cuda/lib"
        ],
        "name": "cupy.fft._callback",
        "sources": [
            "cupy/fft/_callback.pyx"
        ]
    },
    "module_name": "cupy.fft._callback"
}
END: Cython Metadata */

#ifndef PY_SSIZE_T_CLEAN
#define PY_SSIZE_T_CLEAN
#endif /* PY_SSIZE_T_CLEAN */
/* InitLimitedAPI */
#if defined(Py_LIMITED_API)
  #if !defined(CYTHON_LIMITED_API)
  #define CYTHON_LIMITED_API 1
  #endif
#elif defined(CYTHON_LIMITED_API)
  #ifdef _MSC_VER
  #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.")
  #else
  #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.
  #endif
#endif

#include "Python.h"
#ifndef Py_PYTHON_H
    #error Python headers needed to compile C extensions, please install development version of Python.
#elif PY_VERSION_HEX < 0x03080000
    #error Cython requires Python 3.8+.
#else
#define __PYX_ABI_VERSION "3_2_5"
#define CYTHON_HEX_VERSION 0x030205F0
#define CYTHON_FUTURE_DIVISION 1
/* CModulePreamble */
#include <stddef.h>
#ifndef offsetof
  #define offsetof(type, member) ( (size_t) & ((type*)0) -> member )
#endif
#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS)
  #ifndef __stdcall
    #define __stdcall
  #endif
  #ifndef __cdecl
    #define __cdecl
  #endif
  #ifndef __fastcall
    #define __fastcall
  #endif
#endif
#ifndef DL_IMPORT
  #define DL_IMPORT(t) t
#endif
#ifndef DL_EXPORT
  #define DL_EXPORT(t) t
#endif
#define __PYX_COMMA ,
#ifndef PY_LONG_LONG
  #define PY_LONG_LONG LONG_LONG
#endif
#ifndef Py_HUGE_VAL
  #define Py_HUGE_VAL HUGE_VAL
#endif
#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX
#if defined(GRAALVM_PYTHON)
  /* For very preliminary testing purposes. Most variables are set the same as PyPy.
     The existence of this section does not imply that anything works or is even tested */
  #define CYTHON_COMPILING_IN_PYPY 0
  #define CYTHON_COMPILING_IN_CPYTHON 0
  #define CYTHON_COMPILING_IN_LIMITED_API 0
  #define CYTHON_COMPILING_IN_GRAAL 1
  #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
  #undef CYTHON_USE_TYPE_SLOTS
  #define CYTHON_USE_TYPE_SLOTS 0
  #undef CYTHON_USE_TYPE_SPECS
  #define CYTHON_USE_TYPE_SPECS 0
  #undef CYTHON_USE_PYTYPE_LOOKUP
  #define CYTHON_USE_PYTYPE_LOOKUP 0
  #undef CYTHON_USE_PYLIST_INTERNALS
  #define CYTHON_USE_PYLIST_INTERNALS 0
  #undef CYTHON_USE_UNICODE_INTERNALS
  #define CYTHON_USE_UNICODE_INTERNALS 0
  #undef CYTHON_USE_UNICODE_WRITER
  #define CYTHON_USE_UNICODE_WRITER 0
  #undef CYTHON_USE_PYLONG_INTERNALS
  #define CYTHON_USE_PYLONG_INTERNALS 0
  #undef CYTHON_AVOID_BORROWED_REFS
  #define CYTHON_AVOID_BORROWED_REFS 1
  #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
  #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0
  #undef CYTHON_ASSUME_SAFE_MACROS
  #define CYTHON_ASSUME_SAFE_MACROS 0
  #undef CYTHON_ASSUME_SAFE_SIZE
  #define CYTHON_ASSUME_SAFE_SIZE 0
  #undef CYTHON_UNPACK_METHODS
  #define CYTHON_UNPACK_METHODS 0
  #undef CYTHON_FAST_THREAD_STATE
  #define CYTHON_FAST_THREAD_STATE 0
  #undef CYTHON_FAST_GIL
  #define CYTHON_FAST_GIL 0
  #undef CYTHON_METH_FASTCALL
  #define CYTHON_METH_FASTCALL 0
  #undef CYTHON_FAST_PYCALL
  #define CYTHON_FAST_PYCALL 0
  #ifndef CYTHON_PEP487_INIT_SUBCLASS
    #define CYTHON_PEP487_INIT_SUBCLASS 1
  #endif
  #undef CYTHON_PEP489_MULTI_PHASE_INIT
  #define CYTHON_PEP489_MULTI_PHASE_INIT 1
  #undef CYTHON_USE_MODULE_STATE
  #define CYTHON_USE_MODULE_STATE 0
  #undef CYTHON_USE_SYS_MONITORING
  #define CYTHON_USE_SYS_MONITORING 0
  #undef CYTHON_USE_TP_FINALIZE
  #define CYTHON_USE_TP_FINALIZE 0
  #undef CYTHON_USE_AM_SEND
  #define CYTHON_USE_AM_SEND 0
  #undef CYTHON_USE_DICT_VERSIONS
  #define CYTHON_USE_DICT_VERSIONS 0
  #undef CYTHON_USE_EXC_INFO_STACK
  #define CYTHON_USE_EXC_INFO_STACK 1
  #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
    #define CYTHON_UPDATE_DESCRIPTOR_DOC 0
  #endif
  #undef CYTHON_USE_FREELISTS
  #define CYTHON_USE_FREELISTS 0
  #undef CYTHON_IMMORTAL_CONSTANTS
  #define CYTHON_IMMORTAL_CONSTANTS 0
#elif defined(PYPY_VERSION)
  #define CYTHON_COMPILING_IN_PYPY 1
  #define CYTHON_COMPILING_IN_CPYTHON 0
  #define CYTHON_COMPILING_IN_LIMITED_API 0
  #define CYTHON_COMPILING_IN_GRAAL 0
  #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
  #undef CYTHON_USE_TYPE_SLOTS
  #define CYTHON_USE_TYPE_SLOTS 1
  #ifndef CYTHON_USE_TYPE_SPECS
    #define CYTHON_USE_TYPE_SPECS 0
  #endif
  #undef CYTHON_USE_PYTYPE_LOOKUP
  #define CYTHON_USE_PYTYPE_LOOKUP 0
  #undef CYTHON_USE_PYLIST_INTERNALS
  #define CYTHON_USE_PYLIST_INTERNALS 0
  #undef CYTHON_USE_UNICODE_INTERNALS
  #define CYTHON_USE_UNICODE_INTERNALS 0
  #undef CYTHON_USE_UNICODE_WRITER
  #define CYTHON_USE_UNICODE_WRITER 0
  #undef CYTHON_USE_PYLONG_INTERNALS
  #define CYTHON_USE_PYLONG_INTERNALS 0
  #undef CYTHON_AVOID_BORROWED_REFS
  #define CYTHON_AVOID_BORROWED_REFS 1
  #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
  #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1
  #undef CYTHON_ASSUME_SAFE_MACROS
  #define CYTHON_ASSUME_SAFE_MACROS 0
  #ifndef CYTHON_ASSUME_SAFE_SIZE
    #define CYTHON_ASSUME_SAFE_SIZE 1
  #endif
  #undef CYTHON_UNPACK_METHODS
  #define CYTHON_UNPACK_METHODS 0
  #undef CYTHON_FAST_THREAD_STATE
  #define CYTHON_FAST_THREAD_STATE 0
  #undef CYTHON_FAST_GIL
  #define CYTHON_FAST_GIL 0
  #undef CYTHON_METH_FASTCALL
  #define CYTHON_METH_FASTCALL 0
  #undef CYTHON_FAST_PYCALL
  #define CYTHON_FAST_PYCALL 0
  #ifndef CYTHON_PEP487_INIT_SUBCLASS
    #define CYTHON_PEP487_INIT_SUBCLASS 1
  #endif
  #if PY_VERSION_HEX < 0x03090000
    #undef CYTHON_PEP489_MULTI_PHASE_INIT
    #define CYTHON_PEP489_MULTI_PHASE_INIT 0
  #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT)
    #define CYTHON_PEP489_MULTI_PHASE_INIT 1
  #endif
  #undef CYTHON_USE_MODULE_STATE
  #define CYTHON_USE_MODULE_STATE 0
  #undef CYTHON_USE_SYS_MONITORING
  #define CYTHON_USE_SYS_MONITORING 0
  #ifndef CYTHON_USE_TP_FINALIZE
    #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00)
  #endif
  #undef CYTHON_USE_AM_SEND
  #define CYTHON_USE_AM_SEND 0
  #undef CYTHON_USE_DICT_VERSIONS
  #define CYTHON_USE_DICT_VERSIONS 0
  #undef CYTHON_USE_EXC_INFO_STACK
  #define CYTHON_USE_EXC_INFO_STACK 0
  #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
    #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100)
  #endif
  #undef CYTHON_USE_FREELISTS
  #define CYTHON_USE_FREELISTS 0
  #undef CYTHON_IMMORTAL_CONSTANTS
  #define CYTHON_IMMORTAL_CONSTANTS 0
#elif defined(CYTHON_LIMITED_API)
  #ifdef Py_LIMITED_API
    #undef __PYX_LIMITED_VERSION_HEX
    #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API
  #endif
  #define CYTHON_COMPILING_IN_PYPY 0
  #define CYTHON_COMPILING_IN_CPYTHON 0
  #define CYTHON_COMPILING_IN_LIMITED_API 1
  #define CYTHON_COMPILING_IN_GRAAL 0
  #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
  #undef CYTHON_USE_TYPE_SLOTS
  #define CYTHON_USE_TYPE_SLOTS 0
  #undef CYTHON_USE_TYPE_SPECS
  #define CYTHON_USE_TYPE_SPECS 1
  #undef CYTHON_USE_PYTYPE_LOOKUP
  #define CYTHON_USE_PYTYPE_LOOKUP 0
  #undef CYTHON_USE_PYLIST_INTERNALS
  #define CYTHON_USE_PYLIST_INTERNALS 0
  #undef CYTHON_USE_UNICODE_INTERNALS
  #define CYTHON_USE_UNICODE_INTERNALS 0
  #ifndef CYTHON_USE_UNICODE_WRITER
    #define CYTHON_USE_UNICODE_WRITER 0
  #endif
  #undef CYTHON_USE_PYLONG_INTERNALS
  #define CYTHON_USE_PYLONG_INTERNALS 0
  #ifndef CYTHON_AVOID_BORROWED_REFS
    #define CYTHON_AVOID_BORROWED_REFS 0
  #endif
  #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
    #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0
  #endif
  #undef CYTHON_ASSUME_SAFE_MACROS
  #define CYTHON_ASSUME_SAFE_MACROS 0
  #undef CYTHON_ASSUME_SAFE_SIZE
  #define CYTHON_ASSUME_SAFE_SIZE 0
  #undef CYTHON_UNPACK_METHODS
  #define CYTHON_UNPACK_METHODS 0
  #undef CYTHON_FAST_THREAD_STATE
  #define CYTHON_FAST_THREAD_STATE 0
  #undef CYTHON_FAST_GIL
  #define CYTHON_FAST_GIL 0
  #undef CYTHON_METH_FASTCALL
  #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000)
  #undef CYTHON_FAST_PYCALL
  #define CYTHON_FAST_PYCALL 0
  #ifndef CYTHON_PEP487_INIT_SUBCLASS
    #define CYTHON_PEP487_INIT_SUBCLASS 1
  #endif
  #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
    #define CYTHON_PEP489_MULTI_PHASE_INIT 1
  #endif
  #ifndef CYTHON_USE_MODULE_STATE
    #define CYTHON_USE_MODULE_STATE 0
  #endif
  #undef CYTHON_USE_SYS_MONITORING
  #define CYTHON_USE_SYS_MONITORING 0
  #ifndef CYTHON_USE_TP_FINALIZE
    #define CYTHON_USE_TP_FINALIZE 0
  #endif
  #ifndef CYTHON_USE_AM_SEND
    #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000)
  #endif
  #undef CYTHON_USE_DICT_VERSIONS
  #define CYTHON_USE_DICT_VERSIONS 0
  #undef CYTHON_USE_EXC_INFO_STACK
  #define CYTHON_USE_EXC_INFO_STACK 0
  #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
    #define CYTHON_UPDATE_DESCRIPTOR_DOC 0
  #endif
  #ifndef CYTHON_USE_FREELISTS
  #define CYTHON_USE_FREELISTS 1
  #endif
  #undef CYTHON_IMMORTAL_CONSTANTS
  #define CYTHON_IMMORTAL_CONSTANTS 0
#else
  #define CYTHON_COMPILING_IN_PYPY 0
  #define CYTHON_COMPILING_IN_CPYTHON 1
  #define CYTHON_COMPILING_IN_LIMITED_API 0
  #define CYTHON_COMPILING_IN_GRAAL 0
  #ifdef Py_GIL_DISABLED
    #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1
  #else
    #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
  #endif
  #if PY_VERSION_HEX < 0x030A0000
    #undef CYTHON_USE_TYPE_SLOTS
    #define CYTHON_USE_TYPE_SLOTS 1
  #elif !defined(CYTHON_USE_TYPE_SLOTS)
    #define CYTHON_USE_TYPE_SLOTS 1
  #endif
  #ifndef CYTHON_USE_TYPE_SPECS
    #define CYTHON_USE_TYPE_SPECS 0
  #endif
  #ifndef CYTHON_USE_PYTYPE_LOOKUP
    #define CYTHON_USE_PYTYPE_LOOKUP 1
  #endif
  #ifndef CYTHON_USE_PYLONG_INTERNALS
    #define CYTHON_USE_PYLONG_INTERNALS 1
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    #undef CYTHON_USE_PYLIST_INTERNALS
    #define CYTHON_USE_PYLIST_INTERNALS 0
  #elif !defined(CYTHON_USE_PYLIST_INTERNALS)
    #define CYTHON_USE_PYLIST_INTERNALS 1
  #endif
  #ifndef CYTHON_USE_UNICODE_INTERNALS
    #define CYTHON_USE_UNICODE_INTERNALS 1
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2
    #undef CYTHON_USE_UNICODE_WRITER
    #define CYTHON_USE_UNICODE_WRITER 0
  #elif !defined(CYTHON_USE_UNICODE_WRITER)
    #define CYTHON_USE_UNICODE_WRITER 1
  #endif
  #ifndef CYTHON_AVOID_BORROWED_REFS
    #define CYTHON_AVOID_BORROWED_REFS 0
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
    #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1
  #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)
    #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0
  #endif
  #ifndef CYTHON_ASSUME_SAFE_MACROS
    #define CYTHON_ASSUME_SAFE_MACROS 1
  #endif
  #ifndef CYTHON_ASSUME_SAFE_SIZE
    #define CYTHON_ASSUME_SAFE_SIZE 1
  #endif
  #ifndef CYTHON_UNPACK_METHODS
    #define CYTHON_UNPACK_METHODS 1
  #endif
  #ifndef CYTHON_FAST_THREAD_STATE
    #define CYTHON_FAST_THREAD_STATE 1
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    #undef CYTHON_FAST_GIL
    #define CYTHON_FAST_GIL 0
  #elif !defined(CYTHON_FAST_GIL)
    #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6)
  #endif
  #ifndef CYTHON_METH_FASTCALL
    #define CYTHON_METH_FASTCALL 1
  #endif
  #ifndef CYTHON_FAST_PYCALL
    #define CYTHON_FAST_PYCALL 1
  #endif
  #ifndef CYTHON_PEP487_INIT_SUBCLASS
    #define CYTHON_PEP487_INIT_SUBCLASS 1
  #endif
  #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
    #define CYTHON_PEP489_MULTI_PHASE_INIT 1
  #endif
  #ifndef CYTHON_USE_MODULE_STATE
    #define CYTHON_USE_MODULE_STATE 0
  #endif
  #ifndef CYTHON_USE_SYS_MONITORING
    #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1)
  #endif
  #ifndef CYTHON_USE_TP_FINALIZE
    #define CYTHON_USE_TP_FINALIZE 1
  #endif
  #ifndef CYTHON_USE_AM_SEND
    #define CYTHON_USE_AM_SEND 1
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    #undef CYTHON_USE_DICT_VERSIONS
    #define CYTHON_USE_DICT_VERSIONS 0
  #elif !defined(CYTHON_USE_DICT_VERSIONS)
    #define CYTHON_USE_DICT_VERSIONS  (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE)
  #endif
  #ifndef CYTHON_USE_EXC_INFO_STACK
    #define CYTHON_USE_EXC_INFO_STACK 1
  #endif
  #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
    #define CYTHON_UPDATE_DESCRIPTOR_DOC 1
  #endif
  #ifndef CYTHON_USE_FREELISTS
    #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING)
  #endif
  #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000
    #undef CYTHON_IMMORTAL_CONSTANTS
    #define CYTHON_IMMORTAL_CONSTANTS 0  // definitely won't work
  #elif !defined(CYTHON_IMMORTAL_CONSTANTS)
    #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING)
  #endif
#endif
#ifndef CYTHON_COMPRESS_STRINGS
  #define CYTHON_COMPRESS_STRINGS 1
#endif
#ifndef CYTHON_FAST_PYCCALL
#define CYTHON_FAST_PYCCALL  CYTHON_FAST_PYCALL
#endif
#ifndef CYTHON_VECTORCALL
#if CYTHON_COMPILING_IN_LIMITED_API
#define CYTHON_VECTORCALL  (__PYX_LIMITED_VERSION_HEX >= 0x030C0000)
#else
#define CYTHON_VECTORCALL  (CYTHON_FAST_PYCCALL)
#endif
#endif
#if CYTHON_USE_PYLONG_INTERNALS
  #undef SHIFT
  #undef BASE
  #undef MASK
  #ifdef SIZEOF_VOID_P
    enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) };
  #endif
#endif
#ifndef __has_attribute
  #define __has_attribute(x) 0
#endif
#ifndef __has_cpp_attribute
  #define __has_cpp_attribute(x) 0
#endif
#ifndef CYTHON_RESTRICT
  #if defined(__GNUC__)
    #define CYTHON_RESTRICT __restrict__
  #elif defined(_MSC_VER) && _MSC_VER >= 1400
    #define CYTHON_RESTRICT __restrict
  #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
    #define CYTHON_RESTRICT restrict
  #else
    #define CYTHON_RESTRICT
  #endif
#endif
#ifndef CYTHON_UNUSED
  #if defined(__cplusplus)
    /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17
     * but leads to warnings with -pedantic, since it is a C++17 feature */
    #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
      #if __has_cpp_attribute(maybe_unused)
        #define CYTHON_UNUSED [[maybe_unused]]
      #endif
    #endif
  #endif
#endif
#ifndef CYTHON_UNUSED
# if defined(__GNUC__)
#   if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
#     define CYTHON_UNUSED __attribute__ ((__unused__))
#   else
#     define CYTHON_UNUSED
#   endif
# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER))
#   define CYTHON_UNUSED __attribute__ ((__unused__))
# else
#   define CYTHON_UNUSED
# endif
#endif
#ifndef CYTHON_UNUSED_VAR
#  if defined(__cplusplus)
     template<class T> void CYTHON_UNUSED_VAR( const T& ) { }
#  else
#    define CYTHON_UNUSED_VAR(x) (void)(x)
#  endif
#endif
#ifndef CYTHON_MAYBE_UNUSED_VAR
  #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x)
#endif
#ifndef CYTHON_NCP_UNUSED
# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
#  define CYTHON_NCP_UNUSED
# else
#  define CYTHON_NCP_UNUSED CYTHON_UNUSED
# endif
#endif
#ifndef CYTHON_USE_CPP_STD_MOVE
  #if defined(__cplusplus) && (\
    __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600))
    #define CYTHON_USE_CPP_STD_MOVE 1
  #else
    #define CYTHON_USE_CPP_STD_MOVE 0
  #endif
#endif
#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None)
#include <stdint.h>
typedef uintptr_t  __pyx_uintptr_t;
#ifndef CYTHON_FALLTHROUGH
  #if defined(__cplusplus)
    /* for clang __has_cpp_attribute(fallthrough) is true even before C++17
     * but leads to warnings with -pedantic, since it is a C++17 feature */
    #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
      #if __has_cpp_attribute(fallthrough)
        #define CYTHON_FALLTHROUGH [[fallthrough]]
      #endif
    #endif
    #ifndef CYTHON_FALLTHROUGH
      #if __has_cpp_attribute(clang::fallthrough)
        #define CYTHON_FALLTHROUGH [[clang::fallthrough]]
      #elif __has_cpp_attribute(gnu::fallthrough)
        #define CYTHON_FALLTHROUGH [[gnu::fallthrough]]
      #endif
    #endif
  #endif
  #ifndef CYTHON_FALLTHROUGH
    #if __has_attribute(fallthrough)
      #define CYTHON_FALLTHROUGH __attribute__((fallthrough))
    #else
      #define CYTHON_FALLTHROUGH
    #endif
  #endif
  #if defined(__clang__) && defined(__apple_build_version__)
    #if __apple_build_version__ < 7000000
      #undef  CYTHON_FALLTHROUGH
      #define CYTHON_FALLTHROUGH
    #endif
  #endif
#endif
#ifndef Py_UNREACHABLE
  #define Py_UNREACHABLE()  assert(0); abort()
#endif
#ifdef __cplusplus
  template <typename T>
  struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);};
  #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL<type>::value)
#else
  #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0)
#endif
#if CYTHON_COMPILING_IN_PYPY == 1
  #define __PYX_NEED_TP_PRINT_SLOT  (PY_VERSION_HEX < 0x030A0000)
#else
  #define __PYX_NEED_TP_PRINT_SLOT  (PY_VERSION_HEX < 0x03090000)
#endif
#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer))

/* CppInitCode */
#ifndef __cplusplus
  #error "Cython files generated with the C++ option must be compiled with a C++ compiler."
#endif
#ifndef CYTHON_INLINE
  #if defined(__clang__)
    #define CYTHON_INLINE __inline__ __attribute__ ((__unused__))
  #else
    #define CYTHON_INLINE inline
  #endif
#endif
template<typename T>
void __Pyx_call_destructor(T& x) {
    x.~T();
}
template<typename T>
class __Pyx_FakeReference {
  public:
    __Pyx_FakeReference() : ptr(NULL) { }
    __Pyx_FakeReference(const T& ref) : ptr(const_cast<T*>(&ref)) { }
    T *operator->() { return ptr; }
    T *operator&() { return ptr; }
    operator T&() { return *ptr; }
    template<typename U> bool operator ==(const U& other) const { return *ptr == other; }
    template<typename U> bool operator !=(const U& other) const { return *ptr != other; }
    template<typename U> bool operator==(const __Pyx_FakeReference<U>& other) const { return *ptr == *other.ptr; }
    template<typename U> bool operator!=(const __Pyx_FakeReference<U>& other) const { return *ptr != *other.ptr; }
  private:
    T *ptr;
};

/* PythonCompatibility */
#define __PYX_BUILD_PY_SSIZE_T "n"
#define CYTHON_FORMAT_SSIZE_T "z"
#define __Pyx_BUILTIN_MODULE_NAME "builtins"
#define __Pyx_DefaultClassType PyType_Type
#if CYTHON_COMPILING_IN_LIMITED_API
    #ifndef CO_OPTIMIZED
    static int CO_OPTIMIZED;
    #endif
    #ifndef CO_NEWLOCALS
    static int CO_NEWLOCALS;
    #endif
    #ifndef CO_VARARGS
    static int CO_VARARGS;
    #endif
    #ifndef CO_VARKEYWORDS
    static int CO_VARKEYWORDS;
    #endif
    #ifndef CO_ASYNC_GENERATOR
    static int CO_ASYNC_GENERATOR;
    #endif
    #ifndef CO_GENERATOR
    static int CO_GENERATOR;
    #endif
    #ifndef CO_COROUTINE
    static int CO_COROUTINE;
    #endif
#else
    #ifndef CO_COROUTINE
      #define CO_COROUTINE 0x80
    #endif
    #ifndef CO_ASYNC_GENERATOR
      #define CO_ASYNC_GENERATOR 0x200
    #endif
#endif
static int __Pyx_init_co_variables(void);
#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE)
  #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type)
#else
  #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type))
#endif
#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is)
  #define __Pyx_Py_Is(x, y)  Py_Is(x, y)
#else
  #define __Pyx_Py_Is(x, y) ((x) == (y))
#endif
#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone)
  #define __Pyx_Py_IsNone(ob) Py_IsNone(ob)
#else
  #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None)
#endif
#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue)
  #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob)
#else
  #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True)
#endif
#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse)
  #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob)
#else
  #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False)
#endif
#define __Pyx_NoneAsNull(obj)  (__Pyx_Py_IsNone(obj) ? NULL : (obj))
#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY
  #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o)
#else
  #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o)
#endif
#ifndef Py_TPFLAGS_CHECKTYPES
  #define Py_TPFLAGS_CHECKTYPES 0
#endif
#ifndef Py_TPFLAGS_HAVE_INDEX
  #define Py_TPFLAGS_HAVE_INDEX 0
#endif
#ifndef Py_TPFLAGS_HAVE_NEWBUFFER
  #define Py_TPFLAGS_HAVE_NEWBUFFER 0
#endif
#ifndef Py_TPFLAGS_HAVE_FINALIZE
  #define Py_TPFLAGS_HAVE_FINALIZE 0
#endif
#ifndef Py_TPFLAGS_SEQUENCE
  #define Py_TPFLAGS_SEQUENCE 0
#endif
#ifndef Py_TPFLAGS_MAPPING
  #define Py_TPFLAGS_MAPPING 0
#endif
#ifndef Py_TPFLAGS_IMMUTABLETYPE
  #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8)
#endif
#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION
  #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7)
#endif
#ifndef METH_STACKLESS
  #define METH_STACKLESS 0
#endif
#ifndef METH_FASTCALL
  #ifndef METH_FASTCALL
     #define METH_FASTCALL 0x80
  #endif
  typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs);
  typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args,
                                                          Py_ssize_t nargs, PyObject *kwnames);
#else
  #if PY_VERSION_HEX >= 0x030d00A4
  #  define __Pyx_PyCFunctionFast PyCFunctionFast
  #  define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords
  #else
  #  define __Pyx_PyCFunctionFast _PyCFunctionFast
  #  define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords
  #endif
#endif
#if CYTHON_METH_FASTCALL
  #define __Pyx_METH_FASTCALL METH_FASTCALL
  #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast
  #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords
#else
  #define __Pyx_METH_FASTCALL METH_VARARGS
  #define __Pyx_PyCFunction_FastCall PyCFunction
  #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords
#endif
#if CYTHON_VECTORCALL
  #define __pyx_vectorcallfunc vectorcallfunc
  #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET  PY_VECTORCALL_ARGUMENTS_OFFSET
  #define __Pyx_PyVectorcall_NARGS(n)  PyVectorcall_NARGS((size_t)(n))
#else
  #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET  0
  #define __Pyx_PyVectorcall_NARGS(n)  ((Py_ssize_t)(n))
#endif
#if PY_VERSION_HEX >= 0x030900B1
#define __Pyx_PyCFunction_CheckExact(func)  PyCFunction_CheckExact(func)
#else
#define __Pyx_PyCFunction_CheckExact(func)  PyCFunction_Check(func)
#endif
#define __Pyx_CyOrPyCFunction_Check(func)  PyCFunction_Check(func)
#if CYTHON_COMPILING_IN_CPYTHON
#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func)  (((PyCFunctionObject*)(func))->m_ml->ml_meth)
#elif !CYTHON_COMPILING_IN_LIMITED_API
#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func)  PyCFunction_GET_FUNCTION(func)
#endif
#if CYTHON_COMPILING_IN_CPYTHON
#define __Pyx_CyOrPyCFunction_GET_FLAGS(func)  (((PyCFunctionObject*)(func))->m_ml->ml_flags)
static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) {
    return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self;
}
#endif
static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) {
#if CYTHON_COMPILING_IN_LIMITED_API
    return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc;
#else
    return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc;
#endif
}
#define __Pyx_IsSameCFunction(func, cfunc)   __Pyx__IsSameCFunction(func, cfunc)
#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000)
  #define __Pyx_PyType_FromModuleAndSpec(m, s, b)  ((void)m, PyType_FromSpecWithBases(s, b))
  typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *);
#else
  #define __Pyx_PyType_FromModuleAndSpec(m, s, b)  PyType_FromModuleAndSpec(m, s, b)
  #define __Pyx_PyCMethod  PyCMethod
#endif
#ifndef METH_METHOD
  #define METH_METHOD 0x200
#endif
#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc)
  #define PyObject_Malloc(s)   PyMem_Malloc(s)
  #define PyObject_Free(p)     PyMem_Free(p)
  #define PyObject_Realloc(p)  PyMem_Realloc(p)
#endif
#if CYTHON_COMPILING_IN_LIMITED_API
  #define __Pyx_PyFrame_SetLineNumber(frame, lineno)
#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000
  #define __Pyx_PyCode_HasFreeVars(co)  (PyCode_GetNumFree(co) > 0)
  #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno))
#elif CYTHON_COMPILING_IN_GRAAL
  #define __Pyx_PyCode_HasFreeVars(co)  (PyCode_GetNumFree(co) > 0)
  #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno))
#else
  #define __Pyx_PyCode_HasFreeVars(co)  (PyCode_GetNumFree(co) > 0)
  #define __Pyx_PyFrame_SetLineNumber(frame, lineno)  (frame)->f_lineno = (lineno)
#endif
#if CYTHON_COMPILING_IN_LIMITED_API
  #define __Pyx_PyThreadState_Current PyThreadState_Get()
#elif !CYTHON_FAST_THREAD_STATE
  #define __Pyx_PyThreadState_Current PyThreadState_GET()
#elif PY_VERSION_HEX >= 0x030d00A1
  #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked()
#else
  #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet()
#endif
#if CYTHON_USE_MODULE_STATE
static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op)
{
    void *result;
    result = PyModule_GetState(op);
    if (!result)
        Py_FatalError("Couldn't find the module state");
    return result;
}
#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o)
#else
#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global)
#endif
#define __Pyx_PyObject_GetSlot(obj, name, func_ctype)  __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype)
#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype)
#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype)
#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype)
#if CYTHON_USE_TYPE_SLOTS
  #define __Pyx_PyType_GetSlot(type, name, func_ctype)  ((type)->name)
  #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype)
  #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL)
  #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype)
#else
  #define __Pyx_PyType_GetSlot(type, name, func_ctype)  ((func_ctype) PyType_GetSlot((type), Py_##name))
  #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\
    ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\
     (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\
     __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL)
  #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype)
  #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype)
#endif
#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized)
#define __Pyx_PyDict_NewPresized(n)  ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n))
#else
#define __Pyx_PyDict_NewPresized(n)  PyDict_New()
#endif
#define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)
#define __Pyx_PyNumber_InPlaceDivide(x,y)  PyNumber_InPlaceTrueDivide(x,y)
#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS
#define __Pyx_PyDict_GetItemStrWithError(dict, name)  _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash)
static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) {
    PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name);
    if (res == NULL) PyErr_Clear();
    return res;
}
#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000
#define __Pyx_PyDict_GetItemStrWithError  PyDict_GetItemWithError
#define __Pyx_PyDict_GetItemStr           PyDict_GetItem
#else
static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) {
#if CYTHON_COMPILING_IN_PYPY
    return PyDict_GetItem(dict, name);
#else
    PyDictEntry *ep;
    PyDictObject *mp = (PyDictObject*) dict;
    long hash = ((PyStringObject *) name)->ob_shash;
    assert(hash != -1);
    ep = (mp->ma_lookup)(mp, name, hash);
    if (ep == NULL) {
        return NULL;
    }
    return ep->me_value;
#endif
}
#define __Pyx_PyDict_GetItemStr           PyDict_GetItem
#endif
#if CYTHON_USE_TYPE_SLOTS
  #define __Pyx_PyType_GetFlags(tp)   (((PyTypeObject *)tp)->tp_flags)
  #define __Pyx_PyType_HasFeature(type, feature)  ((__Pyx_PyType_GetFlags(type) & (feature)) != 0)
#else
  #define __Pyx_PyType_GetFlags(tp)   (PyType_GetFlags((PyTypeObject *)tp))
  #define __Pyx_PyType_HasFeature(type, feature)  PyType_HasFeature(type, feature)
#endif
#define __Pyx_PyObject_GetIterNextFunc(iterator)  __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc)
#if CYTHON_USE_TYPE_SPECS
#define __Pyx_PyHeapTypeObject_GC_Del(obj)  {\
    PyTypeObject *type = Py_TYPE((PyObject*)obj);\
    assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\
    PyObject_GC_Del(obj);\
    Py_DECREF(type);\
}
#else
#define __Pyx_PyHeapTypeObject_GC_Del(obj)  PyObject_GC_Del(obj)
#endif
#if CYTHON_COMPILING_IN_LIMITED_API
  #define __Pyx_PyUnicode_READY(op)       (0)
  #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i)
  #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u)   ((void)u, 1114111U)
  #define __Pyx_PyUnicode_KIND(u)         ((void)u, (0))
  #define __Pyx_PyUnicode_DATA(u)         ((void*)u)
  #define __Pyx_PyUnicode_READ(k, d, i)   ((void)k, PyUnicode_ReadChar((PyObject*)(d), i))
  #define __Pyx_PyUnicode_IS_TRUE(u)      (0 != PyUnicode_GetLength(u))
#else
  #if PY_VERSION_HEX >= 0x030C0000
    #define __Pyx_PyUnicode_READY(op)       (0)
  #else
    #define __Pyx_PyUnicode_READY(op)       (likely(PyUnicode_IS_READY(op)) ?\
                                                0 : _PyUnicode_Ready((PyObject *)(op)))
  #endif
  #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i)
  #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u)   PyUnicode_MAX_CHAR_VALUE(u)
  #define __Pyx_PyUnicode_KIND(u)         ((int)PyUnicode_KIND(u))
  #define __Pyx_PyUnicode_DATA(u)         PyUnicode_DATA(u)
  #define __Pyx_PyUnicode_READ(k, d, i)   PyUnicode_READ(k, d, i)
  #define __Pyx_PyUnicode_WRITE(k, d, i, ch)  PyUnicode_WRITE(k, d, i, (Py_UCS4) ch)
  #if PY_VERSION_HEX >= 0x030C0000
    #define __Pyx_PyUnicode_IS_TRUE(u)      (0 != PyUnicode_GET_LENGTH(u))
  #else
    #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000
    #define __Pyx_PyUnicode_IS_TRUE(u)      (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length))
    #else
    #define __Pyx_PyUnicode_IS_TRUE(u)      (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u)))
    #endif
  #endif
#endif
#if CYTHON_COMPILING_IN_PYPY
  #define __Pyx_PyUnicode_Concat(a, b)      PyNumber_Add(a, b)
  #define __Pyx_PyUnicode_ConcatSafe(a, b)  PyNumber_Add(a, b)
#else
  #define __Pyx_PyUnicode_Concat(a, b)      PyUnicode_Concat(a, b)
  #define __Pyx_PyUnicode_ConcatSafe(a, b)  ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\
      PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b))
#endif
#if CYTHON_COMPILING_IN_PYPY
  #if !defined(PyUnicode_DecodeUnicodeEscape)
    #define PyUnicode_DecodeUnicodeEscape(s, size, errors)  PyUnicode_Decode(s, size, "unicode_escape", errors)
  #endif
  #if !defined(PyUnicode_Contains)
    #define PyUnicode_Contains(u, s)  PySequence_Contains(u, s)
  #endif
  #if !defined(PyByteArray_Check)
    #define PyByteArray_Check(obj)  PyObject_TypeCheck(obj, &PyByteArray_Type)
  #endif
  #if !defined(PyObject_Format)
    #define PyObject_Format(obj, fmt)  PyObject_CallMethod(obj, "__format__", "O", fmt)
  #endif
#endif
#define __Pyx_PyUnicode_FormatSafe(a, b)  ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b))
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000
  #define __Pyx_PySequence_ListKeepNew(obj)\
    (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj))
#elif CYTHON_COMPILING_IN_CPYTHON
  #define __Pyx_PySequence_ListKeepNew(obj)\
    (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj))
#else
  #define __Pyx_PySequence_ListKeepNew(obj)  PySequence_List(obj)
#endif
#ifndef PySet_CheckExact
  #define PySet_CheckExact(obj)        __Pyx_IS_TYPE(obj, &PySet_Type)
#endif
#if PY_VERSION_HEX >= 0x030900A4
  #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt)
  #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size)
#else
  #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt)
  #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size)
#endif
enum __Pyx_ReferenceSharing {
  __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check
  __Pyx_ReferenceSharing_OwnStrongReference,
  __Pyx_ReferenceSharing_FunctionArgument,
  __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar
};
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000
#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\
    (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\
      (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\
      (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0)))
#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API
#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1)
#else
#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0)
#endif
#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
  #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
    #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i)
  #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS
    #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL))
  #else
    #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i)
  #endif
#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS
  #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
    #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i)
  #else
    #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i))
  #endif
#else
  #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i))
#endif
#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS
  #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\
    __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i))
#else
  #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i)
#endif
#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result)
#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) {
  *result = PyObject_GetItem(dict, key);
  if (*result == NULL) {
    if (PyErr_ExceptionMatches(PyExc_KeyError)) {
      PyErr_Clear();
      return 0;
    }
    return -1;
  }
  return 1;
}
#else
static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) {
  *result = PyDict_GetItemWithError(dict, key);
  if (*result == NULL) {
    return PyErr_Occurred() ? -1 : 0;
  }
  Py_INCREF(*result);
  return 1;
}
#endif
#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST
  #define __Pyx_VISIT_CONST(obj)  Py_VISIT(obj)
#else
  #define __Pyx_VISIT_CONST(obj)
#endif
#if CYTHON_ASSUME_SAFE_MACROS
  #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i)
  #define __Pyx_PySequence_SIZE(seq)  Py_SIZE(seq)
  #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0))
  #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i)
  #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0))
  #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i)
#else
  #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i)
  #define __Pyx_PySequence_SIZE(seq)  PySequence_Size(seq)
  #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v)
  #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i)
  #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v)
  #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i)
#endif
#if CYTHON_ASSUME_SAFE_SIZE
  #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o)
  #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o)
  #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o)
  #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o)
  #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o)
  #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o)
#else
  #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o)
  #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o)
  #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o)
  #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o)
  #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o)
  #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o)
#endif
#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString)
  #define PyUnicode_InternFromString(s) PyUnicode_FromString(s)
#endif
#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t
#define __Pyx_PyLong_AsHash_t   __Pyx_PyIndex_AsSsize_t
#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000
    #define __Pyx_PySendResult PySendResult
#else
    typedef enum {
        PYGEN_RETURN = 0,
        PYGEN_ERROR = -1,
        PYGEN_NEXT = 1,
    } __Pyx_PySendResult;
#endif
#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3
  typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result);
#else
  #define __Pyx_pyiter_sendfunc sendfunc
#endif
#if !CYTHON_USE_AM_SEND
#define __PYX_HAS_PY_AM_SEND 0
#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000
#define __PYX_HAS_PY_AM_SEND 1
#else
#define __PYX_HAS_PY_AM_SEND 2  // our own backported implementation
#endif
#if __PYX_HAS_PY_AM_SEND < 2
    #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods
#else
    typedef struct {
        unaryfunc am_await;
        unaryfunc am_aiter;
        unaryfunc am_anext;
        __Pyx_pyiter_sendfunc am_send;
    } __Pyx_PyAsyncMethodsStruct;
    #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s))
#endif
#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0
    #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21)
#else
    #define __Pyx_TPFLAGS_HAVE_AM_SEND (0)
#endif
#if PY_VERSION_HEX >= 0x03090000
#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get()
#else
#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp
#endif
#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000
#ifdef __cplusplus
extern "C"
#endif
PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
#endif
#if CYTHON_COMPILING_IN_LIMITED_API
static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) {
    int value;
    PyObject *py_value = PyObject_GetAttrString(inspect, name);
    if (!py_value) return 0;
    value = (int) PyLong_AsLong(py_value);
    Py_DECREF(py_value);
    *write_to = value;
    return value != -1 || !PyErr_Occurred();
}
static int __Pyx_init_co_variables(void) {
    PyObject *inspect;
    int result;
    inspect = PyImport_ImportModule("inspect");
    result =
#if !defined(CO_OPTIMIZED)
        __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) &&
#endif
#if !defined(CO_NEWLOCALS)
        __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) &&
#endif
#if !defined(CO_VARARGS)
        __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) &&
#endif
#if !defined(CO_VARKEYWORDS)
        __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) &&
#endif
#if !defined(CO_ASYNC_GENERATOR)
        __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) &&
#endif
#if !defined(CO_GENERATOR)
        __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) &&
#endif
#if !defined(CO_COROUTINE)
        __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) &&
#endif
        1;
    Py_DECREF(inspect);
    return result ? 0 : -1;
}
#else
static int __Pyx_init_co_variables(void) {
    return 0;  // It's a limited API-only feature
}
#endif

/* MathInitCode */
#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)
  #ifndef _USE_MATH_DEFINES
    #define _USE_MATH_DEFINES
  #endif
#endif
#include <math.h>
#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL)
#define __Pyx_truncl trunc
#else
#define __Pyx_truncl truncl
#endif

#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME
#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0
#endif
#ifndef CYTHON_CLINE_IN_TRACEBACK
#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME
#endif
#if CYTHON_CLINE_IN_TRACEBACK
#define __PYX_MARK_ERR_POS(f_index, lineno)  { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; }
#else
#define __PYX_MARK_ERR_POS(f_index, lineno)  { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; }
#endif
#define __PYX_ERR(f_index, lineno, Ln_error) \
    { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; }

#ifdef CYTHON_EXTERN_C
    #undef __PYX_EXTERN_C
    #define __PYX_EXTERN_C CYTHON_EXTERN_C
#elif defined(__PYX_EXTERN_C)
    #ifdef _MSC_VER
    #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.")
    #else
    #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.
    #endif
#else
    #define __PYX_EXTERN_C extern "C++"
#endif

#define __PYX_HAVE__cupy__fft___callback
#define __PYX_HAVE_API__cupy__fft___callback
/* Early includes */
#include <stdint.h>
#include "ios"
#include "new"
#include "stdexcept"
#include "typeinfo"
#include <vector>
#include <utility>

    #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)
    // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully
    #include <type_traits>

    namespace cython_std {
    template <typename T> typename std::remove_reference<T>::type&& move(T& t) noexcept { return std::move(t); }
    template <typename T> typename std::remove_reference<T>::type&& move(T&& t) noexcept { return std::move(t); }
    }

    #endif
    
#include <map>
#ifdef _OPENMP
#include <omp.h>
#endif /* _OPENMP */

#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS)
#define CYTHON_WITHOUT_ASSERTIONS
#endif

#ifdef CYTHON_FREETHREADING_COMPATIBLE
#if CYTHON_FREETHREADING_COMPATIBLE
#define __Pyx_FREETHREADING_COMPATIBLE Py_MOD_GIL_NOT_USED
#else
#define __Pyx_FREETHREADING_COMPATIBLE Py_MOD_GIL_USED
#endif
#else
#define __Pyx_FREETHREADING_COMPATIBLE Py_MOD_GIL_NOT_USED
#endif
#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0
#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0
#define __PYX_DEFAULT_STRING_ENCODING ""
#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString
#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
#define __Pyx_uchar_cast(c) ((unsigned char)c)
#define __Pyx_long_cast(x) ((long)x)
#define __Pyx_fits_Py_ssize_t(v, type, is_signed)  (\
    (sizeof(type) < sizeof(Py_ssize_t))  ||\
    (sizeof(type) > sizeof(Py_ssize_t) &&\
          likely(v < (type)PY_SSIZE_T_MAX ||\
                 v == (type)PY_SSIZE_T_MAX)  &&\
          (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\
                                v == (type)PY_SSIZE_T_MIN)))  ||\
    (sizeof(type) == sizeof(Py_ssize_t) &&\
          (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\
                               v == (type)PY_SSIZE_T_MAX)))  )
static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) {
    return (size_t) i < (size_t) limit;
}
#if defined (__cplusplus) && __cplusplus >= 201103L
    #include <cstdlib>
    #define __Pyx_sst_abs(value) std::abs(value)
#elif SIZEOF_INT >= SIZEOF_SIZE_T
    #define __Pyx_sst_abs(value) abs(value)
#elif SIZEOF_LONG >= SIZEOF_SIZE_T
    #define __Pyx_sst_abs(value) labs(value)
#elif defined (_MSC_VER)
    #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value))
#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
    #define __Pyx_sst_abs(value) llabs(value)
#elif defined (__GNUC__)
    #define __Pyx_sst_abs(value) __builtin_llabs(value)
#else
    #define __Pyx_sst_abs(value) ((value<0) ? -value : value)
#endif
static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s);
static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*);
static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length);
static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*);
#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l)
#define __Pyx_PyBytes_FromString        PyBytes_FromString
#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize
static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*);
#if CYTHON_ASSUME_SAFE_MACROS
    #define __Pyx_PyBytes_AsWritableString(s)     ((char*) PyBytes_AS_STRING(s))
    #define __Pyx_PyBytes_AsWritableSString(s)    ((signed char*) PyBytes_AS_STRING(s))
    #define __Pyx_PyBytes_AsWritableUString(s)    ((unsigned char*) PyBytes_AS_STRING(s))
    #define __Pyx_PyBytes_AsString(s)     ((const char*) PyBytes_AS_STRING(s))
    #define __Pyx_PyBytes_AsSString(s)    ((const signed char*) PyBytes_AS_STRING(s))
    #define __Pyx_PyBytes_AsUString(s)    ((const unsigned char*) PyBytes_AS_STRING(s))
    #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s)
#else
    #define __Pyx_PyBytes_AsWritableString(s)     ((char*) PyBytes_AsString(s))
    #define __Pyx_PyBytes_AsWritableSString(s)    ((signed char*) PyBytes_AsString(s))
    #define __Pyx_PyBytes_AsWritableUString(s)    ((unsigned char*) PyBytes_AsString(s))
    #define __Pyx_PyBytes_AsString(s)     ((const char*) PyBytes_AsString(s))
    #define __Pyx_PyBytes_AsSString(s)    ((const signed char*) PyBytes_AsString(s))
    #define __Pyx_PyBytes_AsUString(s)    ((const unsigned char*) PyBytes_AsString(s))
    #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s)
#endif
#define __Pyx_PyObject_AsWritableString(s)    ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s))
#define __Pyx_PyObject_AsWritableSString(s)    ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s))
#define __Pyx_PyObject_AsWritableUString(s)    ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s))
#define __Pyx_PyObject_AsSString(s)    ((const signed char*) __Pyx_PyObject_AsString(s))
#define __Pyx_PyObject_AsUString(s)    ((const unsigned char*) __Pyx_PyObject_AsString(s))
#define __Pyx_PyObject_FromCString(s)  __Pyx_PyObject_FromString((const char*)s)
#define __Pyx_PyBytes_FromCString(s)   __Pyx_PyBytes_FromString((const char*)s)
#define __Pyx_PyByteArray_FromCString(s)   __Pyx_PyByteArray_FromString((const char*)s)
#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s)
#define __Pyx_PyUnicode_FromOrdinal(o)       PyUnicode_FromOrdinal((int)o)
#define __Pyx_PyUnicode_AsUnicode            PyUnicode_AsUnicode
static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) {
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef)
    return Py_NewRef(obj);
#else
    Py_INCREF(obj);
    return obj;
#endif
}
static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) {
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef)
    return Py_XNewRef(obj);
#else
    Py_XINCREF(obj);
    return obj;
#endif
}
static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b);
static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b);
static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*);
static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*);
static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x);
#define __Pyx_PySequence_Tuple(obj)\
    (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj))
static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*);
static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t);
static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*);
#if CYTHON_ASSUME_SAFE_MACROS
#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x))
#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x)
#else
#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x)
#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x)
#endif
#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x))
#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x))
#if CYTHON_USE_PYLONG_INTERNALS
  #if PY_VERSION_HEX >= 0x030C00A7
  #ifndef _PyLong_SIGN_MASK
    #define _PyLong_SIGN_MASK 3
  #endif
  #ifndef _PyLong_NON_SIZE_BITS
    #define _PyLong_NON_SIZE_BITS 3
  #endif
  #define __Pyx_PyLong_Sign(x)  (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK)
  #define __Pyx_PyLong_IsNeg(x)  ((__Pyx_PyLong_Sign(x) & 2) != 0)
  #define __Pyx_PyLong_IsNonNeg(x)  (!__Pyx_PyLong_IsNeg(x))
  #define __Pyx_PyLong_IsZero(x)  (__Pyx_PyLong_Sign(x) & 1)
  #define __Pyx_PyLong_IsPos(x)  (__Pyx_PyLong_Sign(x) == 0)
  #define __Pyx_PyLong_CompactValueUnsigned(x)  (__Pyx_PyLong_Digits(x)[0])
  #define __Pyx_PyLong_DigitCount(x)  ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS))
  #define __Pyx_PyLong_SignedDigitCount(x)\
        ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x))
  #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue)
    #define __Pyx_PyLong_IsCompact(x)     PyUnstable_Long_IsCompact((PyLongObject*) x)
    #define __Pyx_PyLong_CompactValue(x)  PyUnstable_Long_CompactValue((PyLongObject*) x)
  #else
    #define __Pyx_PyLong_IsCompact(x)     (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS))
    #define __Pyx_PyLong_CompactValue(x)  ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0])
  #endif
  typedef Py_ssize_t  __Pyx_compact_pylong;
  typedef size_t  __Pyx_compact_upylong;
  #else
  #define __Pyx_PyLong_IsNeg(x)  (Py_SIZE(x) < 0)
  #define __Pyx_PyLong_IsNonNeg(x)  (Py_SIZE(x) >= 0)
  #define __Pyx_PyLong_IsZero(x)  (Py_SIZE(x) == 0)
  #define __Pyx_PyLong_IsPos(x)  (Py_SIZE(x) > 0)
  #define __Pyx_PyLong_CompactValueUnsigned(x)  ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0])
  #define __Pyx_PyLong_DigitCount(x)  __Pyx_sst_abs(Py_SIZE(x))
  #define __Pyx_PyLong_SignedDigitCount(x)  Py_SIZE(x)
  #define __Pyx_PyLong_IsCompact(x)  (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1)
  #define __Pyx_PyLong_CompactValue(x)\
        ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0]))
  typedef sdigit  __Pyx_compact_pylong;
  typedef digit  __Pyx_compact_upylong;
  #endif
  #if PY_VERSION_HEX >= 0x030C00A5
  #define __Pyx_PyLong_Digits(x)  (((PyLongObject*)x)->long_value.ob_digit)
  #else
  #define __Pyx_PyLong_Digits(x)  (((PyLongObject*)x)->ob_digit)
  #endif
#endif
#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8
  #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL)
#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
  #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL)
#else
  #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL)
#endif


/* Test for GCC > 2.95 */
#if defined(__GNUC__)     && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  #define likely(x)   __builtin_expect(!!(x), 1)
  #define unlikely(x) __builtin_expect(!!(x), 0)
#else /* !__GNUC__ or GCC < 2.95 */
  #define likely(x)   (x)
  #define unlikely(x) (x)
#endif /* __GNUC__ */
/* PretendToInitialize */
#ifdef __cplusplus
#if __cplusplus > 201103L
#include <type_traits>
#endif
template <typename T>
static void __Pyx_pretend_to_initialize(T* ptr) {
#if __cplusplus > 201103L
    if ((std::is_trivially_default_constructible<T>::value))
#endif
        *ptr = T();
    (void)ptr;
}
#else
static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }
#endif


#if !CYTHON_USE_MODULE_STATE
static PyObject *__pyx_m = NULL;
#endif
static int __pyx_lineno;
static int __pyx_clineno = 0;
static const char * const __pyx_cfilenm = __FILE__;
static const char *__pyx_filename;

/* #### Code section: filename_table ### */

static const char* const __pyx_f[] = {
  "cupy/fft/_callback.pyx",
  "cupy/cuda/memory.pxd",
  "cupy/_core/core.pxd",
  "(tree fragment)",
  "cupy_backends/cuda/api/runtime.pxd",
  "cupy/cuda/device.pxd",
  "cupy/cuda/function.pxd",
  "cupy/_core/_carray.pxd",
};
/* #### Code section: utility_code_proto_before_types ### */
/* CriticalSectionsDefinition.proto (used by CriticalSections) */
#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
#define __Pyx_PyCriticalSection void*
#define __Pyx_PyCriticalSection2 void*
#define __Pyx_PyCriticalSection_End(cs)
#define __Pyx_PyCriticalSection2_End(cs)
#else
#define __Pyx_PyCriticalSection PyCriticalSection
#define __Pyx_PyCriticalSection2 PyCriticalSection2
#define __Pyx_PyCriticalSection_End PyCriticalSection_End
#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End
#endif

/* CriticalSections.proto */
#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs)
#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs)
#else
#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin
#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin
#endif
#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API
#define __Pyx_BEGIN_CRITICAL_SECTION(o) {
#define __Pyx_END_CRITICAL_SECTION() }
#else
#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION
#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION
#endif

/* Atomics.proto (used by UnpackUnboundCMethod) */
#include <pythread.h>
#ifndef CYTHON_ATOMICS
    #define CYTHON_ATOMICS 1
#endif
#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS
#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
#define __pyx_atomic_int_type int
#define __pyx_nonatomic_int_type int
#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\
                        (__STDC_VERSION__ >= 201112L) &&\
                        !defined(__STDC_NO_ATOMICS__))
    #include <stdatomic.h>
#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\
                    (__cplusplus >= 201103L) ||\
                    (defined(_MSC_VER) && _MSC_VER >= 1700)))
    #include <atomic>
#endif
#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\
                        (__STDC_VERSION__ >= 201112L) &&\
                        !defined(__STDC_NO_ATOMICS__) &&\
                       ATOMIC_INT_LOCK_FREE == 2)
    #undef __pyx_atomic_int_type
    #define __pyx_atomic_int_type atomic_int
    #define __pyx_atomic_ptr_type atomic_uintptr_t
    #define __pyx_nonatomic_ptr_type uintptr_t
    #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed)
    #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel)
    #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel)
    #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg)
    #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired)
    #define __pyx_atomic_load(value) atomic_load(value)
    #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value)
    #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed)
    #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire)
    #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value)
    #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired)
    #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER)
        #pragma message ("Using standard C atomics")
    #elif defined(__PYX_DEBUG_ATOMICS)
        #warning "Using standard C atomics"
    #endif
#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\
                    (__cplusplus >= 201103L) ||\
\
                    (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\
                    ATOMIC_INT_LOCK_FREE == 2)
    #undef __pyx_atomic_int_type
    #define __pyx_atomic_int_type std::atomic_int
    #define __pyx_atomic_ptr_type std::atomic_uintptr_t
    #define __pyx_nonatomic_ptr_type uintptr_t
    #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed)
    #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel)
    #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel)
    #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg)
    #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired)
    #define __pyx_atomic_load(value) std::atomic_load(value)
    #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value)
    #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed)
    #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire)
    #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value)
    #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired)
    #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER)
        #pragma message ("Using standard C++ atomics")
    #elif defined(__PYX_DEBUG_ATOMICS)
        #warning "Using standard C++ atomics"
    #endif
#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\
                    (__GNUC_MINOR__ > 1 ||\
                    (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2))))
    #define __pyx_atomic_ptr_type void*
    #define __pyx_nonatomic_ptr_type void*
    #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1)
    #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1)
    #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1)
    #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg)
    static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) {
        __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired);
        int result = old == *expected;
        *expected = old;
        return result;
    }
    #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0)
    #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value)
    #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0)
    #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0)
    #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value)
    static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) {
        __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired);
        int result = old == *expected;
        *expected = old;
        return result;
    }
    #ifdef __PYX_DEBUG_ATOMICS
        #warning "Using GNU atomics"
    #endif
#elif CYTHON_ATOMICS && defined(_MSC_VER)
    #include <intrin.h>
    #undef __pyx_atomic_int_type
    #define __pyx_atomic_int_type long
    #define __pyx_atomic_ptr_type void*
    #undef __pyx_nonatomic_int_type
    #define __pyx_nonatomic_int_type long
    #define __pyx_nonatomic_ptr_type void*
    #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer)
    #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1)
    #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1)
    #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1)
    #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg)
    static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) {
        __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected);
        int result = old == *expected;
        *expected = old;
        return result;
    }
    #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0)
    #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value)
    #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value
    #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0)
    #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value)
    static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) {
        __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected);
        int result = old == *expected;
        *expected = old;
        return result;
    }
    #ifdef __PYX_DEBUG_ATOMICS
        #pragma message ("Using MSVC atomics")
    #endif
#else
    #undef CYTHON_ATOMICS
    #define CYTHON_ATOMICS 0
    #ifdef __PYX_DEBUG_ATOMICS
        #warning "Not using atomics"
    #endif
#endif

/* IncludeStructmemberH.proto (used by FixUpExtensionType) */
#include <structmember.h>

/* #### Code section: numeric_typedefs ### */
/* #### Code section: complex_type_declarations ### */
/* #### Code section: type_declarations ### */

/*--- Type declarations ---*/
struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes;
struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps;
struct __pyx_obj_4cupy_4cuda_6device_Handle;
struct __pyx_obj_4cupy_4cuda_6device_Device;
struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory;
struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory;
struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer;
struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool;
struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory;
struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator;
struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory;
struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator;
struct __pyx_obj_4cupy_4cuda_8function_CPointer;
struct __pyx_obj_4cupy_4cuda_8function_Function;
struct __pyx_obj_4cupy_4cuda_8function_Module;
struct __pyx_obj_4cupy_4cuda_8function_LinkState;
struct __pyx_obj_4cupy_5_core_7_carray_mdspan;
struct __pyx_obj_4cupy_5_core_7_carray_CArray;
struct __pyx_obj_4cupy_5_core_7_carray_CIndexer;
struct __pyx_obj_4cupy_5_core_7_carray_Indexer;
struct __pyx_obj_4cupy_5_core_4core__ndarray_base;
struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal;
struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager;
struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager;
struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager;
struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks;

/* "cupy_backends/cuda/api/_driver_enum.pxd":1
 * cpdef enum:             # <<<<<<<<<<<<<<
 *     # CUjitInputType
 *     CU_JIT_INPUT_CUBIN = 0
*/
enum  {
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INPUT_CUBIN = 0,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INPUT_PTX = 1,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INPUT_FATBINARY = 2,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INPUT_OBJECT = 3,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INPUT_LIBRARY = 4,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INPUT_NVVM = 5,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_MAX_REGISTERS = 0,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_THREADS_PER_BLOCK = 1,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_WALL_TIME = 2,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INFO_LOG_BUFFER = 3,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES = 4,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_ERROR_LOG_BUFFER = 5,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES = 6,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_OPTIMIZATION_LEVEL = 7,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_TARGET_FROM_CUCONTEXT = 8,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_TARGET = 9,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_FALLBACK_STRATEGY = 10,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_GENERATE_DEBUG_INFO = 11,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_LOG_VERBOSE = 12,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_GENERATE_LINE_INFO = 13,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_CACHE_MODE = 14,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_NEW_SM3X_OPT = 15,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_FAST_COMPILE = 16,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_GLOBAL_SYMBOL_NAMES = 17,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_GLOBAL_SYMBOL_ADDRESSES = 18,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_GLOBAL_SYMBOL_COUNT = 19,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_LTO = 20,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_FTZ = 21,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_PREC_DIV = 22,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_PREC_SQRT = 23,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_FMA = 24,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_JIT_NUM_OPTIONS = 25,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK = 0,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES = 1,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES = 2,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES = 3,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_NUM_REGS = 4,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_PTX_VERSION = 5,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_BINARY_VERSION = 6,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_CACHE_MODE_CA = 7,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES = 8,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 9,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CUDA_ERROR_INVALID_VALUE = 1,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_UNSIGNED_INT8 = 0x01,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_UNSIGNED_INT16 = 0x02,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_UNSIGNED_INT32 = 0x03,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_SIGNED_INT8 = 0x08,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_SIGNED_INT16 = 0x09,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_SIGNED_INT32 = 0x0a,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_HALF = 0x10,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_AD_FORMAT_FLOAT = 0x20,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TR_ADDRESS_MODE_WRAP = 0,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TR_ADDRESS_MODE_CLAMP = 1,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TR_ADDRESS_MODE_MIRROR = 2,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TR_ADDRESS_MODE_BORDER = 3,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TR_FILTER_MODE_POINT = 0,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TR_FILTER_MODE_LINEAR = 1,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TRSA_OVERRIDE_FORMAT = 0x01,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TRSF_READ_AS_INTEGER = 0x01,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TRSF_NORMALIZED_COORDINATES = 0x02,
  __pyx_e_13cupy_backends_4cuda_3api_12_driver_enum_CU_TRSF_SRGB = 0x10
};

/* "cupy_backends/cuda/api/_runtime_enum.pxd":1
 * cpdef enum:             # <<<<<<<<<<<<<<
 *     # need to revisit this when cython supports C++ enums (in 3.0)
 *     # https://stackoverflow.com/a/67138945
*/
enum  {

  /* "cupy_backends/cuda/api/_runtime_enum.pxd":116
 *     cudaGraphDebugDotFlagsKernelNodeAttributes = 1<<9
 *     cudaGraphDebugDotFlagsHandles = 1<<10
 *     cudaGraphDebugDotFlagsConditionalNodeParams = 1<<15             # <<<<<<<<<<<<<<
 * 
 * # This was a legacy mistake: the prefix "cuda" should have been removed
*/
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemoryTypeHost = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemoryTypeDevice = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaIpcMemLazyEnablePeerAccess = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAttachGlobal = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAttachHost = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAttachSingle = 4,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaCpuDeviceId = -1L,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaInvalidDeviceId = -2L,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAdviseSetReadMostly = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAdviseUnsetReadMostly = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAdviseSetPreferredLocation = 3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAdviseUnsetPreferredLocation = 4,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAdviseSetAccessedBy = 5,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAdviseUnsetAccessedBy = 6,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_R_32F = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_R_64F = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_R_16F = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_R_8I = 3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_C_32F = 4,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_C_64F = 5,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_C_16F = 6,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_C_8I = 7,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_R_8U = 8,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_C_8U = 9,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_CUDA_R_16BF = 14,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaLimitStackSize = 0x00,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaLimitPrintfFifoSize = 0x01,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaLimitMallocHeapSize = 0x02,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaLimitDevRuntimeSyncDepth = 0x03,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaLimitDevRuntimePendingLaunchCount = 0x04,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaLimitMaxL2FetchGranularity = 0x05,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaChannelFormatKindSigned = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaChannelFormatKindUnsigned = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaChannelFormatKindFloat = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaChannelFormatKindNone = 3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaArrayDefault = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaArraySurfaceLoadStore = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaResourceTypeArray = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaResourceTypeMipmappedArray = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaResourceTypeLinear = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaResourceTypePitch2D = 3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaAddressModeWrap = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaAddressModeClamp = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaAddressModeMirror = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaAddressModeBorder = 3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaFilterModePoint = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaFilterModeLinear = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaReadModeElementType = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaReadModeNormalizedFloat = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolReuseFollowEventDependencies = 0x1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolReuseAllowOpportunistic = 0x2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolReuseAllowInternalDependencies = 0x3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolAttrReleaseThreshold = 0x4,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolAttrReservedMemCurrent = 0x5,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolAttrReservedMemHigh = 0x6,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolAttrUsedMemCurrent = 0x7,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemPoolAttrUsedMemHigh = 0x8,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemAllocationTypePinned = 0x1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemHandleTypeNone = 0x0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemHandleTypePosixFileDescriptor = 0x1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemLocationTypeInvalid = 0,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemLocationTypeDevice = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemLocationTypeHost = 2,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemLocationTypeHostNuma = 3,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaMemLocationTypeHostNumaCurrent = 4,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsVerbose = (1 << 0),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsKernelNodeParams = (1 << 2),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsMemcpyNodeParams = (1 << 3),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsMemsetNodeParams = (1 << 4),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsHostNodeParams = (1 << 5),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsEventNodeParams = (1 << 6),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsExtSemasSignalNodeParams = (1 << 7),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsExtSemasWaitNodeParams = (1 << 8),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsKernelNodeAttributes = (1 << 9),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsHandles = (1 << 10),
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaGraphDebugDotFlagsConditionalNodeParams = (1 << 15)
};

/* "cupy_backends/cuda/api/_runtime_enum.pxd":250
 * ELSE:
 *     # For CUDA/RTD
 *     cpdef enum:             # <<<<<<<<<<<<<<
 *         cudaDevAttrMaxThreadsPerBlock = 1
 *         cudaDevAttrMaxBlockDimX
*/
enum  {
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxThreadsPerBlock = 1,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxBlockDimX,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxBlockDimY,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxBlockDimZ,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxGridDimX,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxGridDimY,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxGridDimZ,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSharedMemoryPerBlock,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrTotalConstantMemory,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrWarpSize,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxPitch,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxRegistersPerBlock,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrClockRate,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrTextureAlignment,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrGpuOverlap,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMultiProcessorCount,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrKernelExecTimeout,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrIntegrated,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrCanMapHostMemory,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrComputeMode,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture1DWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture3DWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture3DHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture3DDepth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DLayeredWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DLayeredHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DLayeredLayers,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrSurfaceAlignment,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrConcurrentKernels,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrEccEnabled,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrPciBusId,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrPciDeviceId,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrTccDriver,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMemoryClockRate,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrGlobalMemoryBusWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrL2CacheSize,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxThreadsPerMultiProcessor,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrAsyncEngineCount,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrUnifiedAddressing,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture1DLayeredWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture1DLayeredLayers,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DGatherWidth = 45,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DGatherHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture3DWidthAlt,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture3DHeightAlt,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture3DDepthAlt,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrPciDomainId,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrTexturePitchAlignment,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTextureCubemapWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTextureCubemapLayeredWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTextureCubemapLayeredLayers,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface1DWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface2DWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface2DHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface3DWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface3DHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface3DDepth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface1DLayeredWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface1DLayeredLayers,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface2DLayeredWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface2DLayeredHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurface2DLayeredLayers,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurfaceCubemapWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurfaceCubemapLayeredWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSurfaceCubemapLayeredLayers,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture1DLinearWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DLinearWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DLinearHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DLinearPitch,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DMipmappedWidth,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture2DMipmappedHeight,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTexture1DMipmappedWidth = 77,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrStreamPrioritiesSupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrGlobalL1CacheSupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrLocalL1CacheSupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSharedMemoryPerMultiprocessor,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxRegistersPerMultiprocessor,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrManagedMemory,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrIsMultiGpuBoard,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMultiGpuBoardGroupID,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrHostNativeAtomicSupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrSingleToDoublePrecisionPerfRatio,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrPageableMemoryAccess,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrConcurrentManagedAccess,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrComputePreemptionSupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrCanUseHostPointerForRegisteredMem,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrReserved92,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrReserved93,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrReserved94,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrCooperativeLaunch,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrCooperativeMultiDeviceLaunch,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxSharedMemoryPerBlockOptin,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrCanFlushRemoteWrites,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrHostRegisterSupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrPageableMemoryAccessUsesHostPageTables,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrDirectManagedMemAccessFromHost,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxBlocksPerMultiprocessor = 0x6A,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrReservedSharedMemoryPerBlock = 0x6F,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrSparseCudaArraySupported = 0x70,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrHostRegisterReadOnlySupported = 0x71,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMaxTimelineSemaphoreInteropSupported = 0x72,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMemoryPoolsSupported = 0x73,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrGPUDirectRDMASupported,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrGPUDirectRDMAFlushWritesOptions,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrGPUDirectRDMAWritesOrdering,
  __pyx_e_13cupy_backends_4cuda_3api_13_runtime_enum_cudaDevAttrMemoryPoolSupportedHandleTypes
};
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_mallocManaged;
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_malloc3DArray;
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_mallocArray;
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_streamAddCallback;
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_streamWaitEvent;
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_streamBeginCapture;

/* "runtime.pxd":99
 * 
 * # For backward compatibility, keep APIs not prefixed with "cuda".
 * cpdef enum:             # <<<<<<<<<<<<<<
 *     memcpyHostToHost = 0
 *     memcpyHostToDevice = 1
*/
enum  {
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memcpyHostToHost = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memcpyHostToDevice = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memcpyDeviceToHost = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memcpyDeviceToDevice = 3,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memcpyDefault = 4,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_hostAllocDefault = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_hostAllocPortable = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_hostAllocMapped = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_hostAllocWriteCombined = 4,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamDefault = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamNonBlocking = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamLegacy = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamPerThread = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_eventDefault = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_eventBlockingSync = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_eventDisableTiming = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_eventInterprocess = 4,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamCaptureModeGlobal = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamCaptureModeThreadLocal = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamCaptureModeRelaxed = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamCaptureStatusNone = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamCaptureStatusActive = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_streamCaptureStatusInvalidated = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memoryTypeUnregistered = 0,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memoryTypeHost = 1,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memoryTypeDevice = 2,
  __pyx_e_13cupy_backends_4cuda_3api_7runtime_memoryTypeManaged = 3
};

/* "runtime.pxd":205
 * 
 * cpdef intptr_t malloc(size_t size) except? 0
 * cpdef intptr_t mallocManaged(size_t size, unsigned int flags=*) except? 0             # <<<<<<<<<<<<<<
 * cpdef intptr_t malloc3DArray(intptr_t desc, size_t width, size_t height,
 *                              size_t depth, unsigned int flags=*) except? 0
*/
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_mallocManaged {
  int __pyx_n;
  unsigned int flags;
};

/* "runtime.pxd":206
 * cpdef intptr_t malloc(size_t size) except? 0
 * cpdef intptr_t mallocManaged(size_t size, unsigned int flags=*) except? 0
 * cpdef intptr_t malloc3DArray(intptr_t desc, size_t width, size_t height,             # <<<<<<<<<<<<<<
 *                              size_t depth, unsigned int flags=*) except? 0
 * cpdef intptr_t mallocArray(intptr_t desc, size_t width, size_t height,
*/
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_malloc3DArray {
  int __pyx_n;
  unsigned int flags;
};

/* "runtime.pxd":208
 * cpdef intptr_t malloc3DArray(intptr_t desc, size_t width, size_t height,
 *                              size_t depth, unsigned int flags=*) except? 0
 * cpdef intptr_t mallocArray(intptr_t desc, size_t width, size_t height,             # <<<<<<<<<<<<<<
 *                            unsigned int flags=*) except? 0
 * cpdef intptr_t mallocAsync(size_t size, intptr_t stream) except? 0
*/
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_mallocArray {
  int __pyx_n;
  unsigned int flags;
};

/* "runtime.pxd":275
 * cpdef streamDestroy(intptr_t stream)
 * cpdef streamSynchronize(intptr_t stream)
 * cpdef streamAddCallback(intptr_t stream, callback, intptr_t arg,             # <<<<<<<<<<<<<<
 *                         unsigned int flags=*)
 * cpdef launchHostFunc(intptr_t stream, callback, intptr_t arg)
*/
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_streamAddCallback {
  int __pyx_n;
  unsigned int flags;
};

/* "runtime.pxd":280
 * cpdef _launchHostFuncUnmanaged(intptr_t stream, tuple func_arg)
 * cpdef streamQuery(intptr_t stream)
 * cpdef streamWaitEvent(intptr_t stream, intptr_t event, unsigned int flags=*)             # <<<<<<<<<<<<<<
 * cpdef streamBeginCapture(intptr_t stream, int mode=*)
 * cpdef intptr_t streamEndCapture(intptr_t stream) except? 0
*/
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_streamWaitEvent {
  int __pyx_n;
  unsigned int flags;
};

/* "runtime.pxd":281
 * cpdef streamQuery(intptr_t stream)
 * cpdef streamWaitEvent(intptr_t stream, intptr_t event, unsigned int flags=*)
 * cpdef streamBeginCapture(intptr_t stream, int mode=*)             # <<<<<<<<<<<<<<
 * cpdef intptr_t streamEndCapture(intptr_t stream) except? 0
 * cpdef bint streamIsCapturing(intptr_t stream) except*
*/
struct __pyx_opt_args_13cupy_backends_4cuda_3api_7runtime_streamBeginCapture {
  int __pyx_n;
  int mode;
};
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_device_async;
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_host_async;
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_async;
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_to_host_async;
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_memset_async;
struct __pyx_opt_args_4cupy_4cuda_6memory_set_allocator;
struct __pyx_opt_args_4cupy_4cuda_6memory_10MemoryPool_free_all_blocks;
struct __pyx_opt_args_4cupy_4cuda_6memory_10MemoryPool_set_limit;

/* "memory.pxd":39
 * 
 *     cpdef copy_from_device(self, MemoryPointer src, size_t size)
 *     cpdef copy_from_device_async(self, MemoryPointer src, size_t size,             # <<<<<<<<<<<<<<
 *                                  stream=?)
 *     cpdef copy_from_host(self, mem, size_t size)
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_device_async {
  int __pyx_n;
  PyObject *stream;
};

/* "memory.pxd":42
 *                                  stream=?)
 *     cpdef copy_from_host(self, mem, size_t size)
 *     cpdef copy_from_host_async(self, mem, size_t size, stream=?)             # <<<<<<<<<<<<<<
 *     cpdef copy_from(self, mem, size_t size)
 *     cpdef copy_from_async(self, mem, size_t size, stream=?)
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_host_async {
  int __pyx_n;
  PyObject *stream;
};

/* "memory.pxd":44
 *     cpdef copy_from_host_async(self, mem, size_t size, stream=?)
 *     cpdef copy_from(self, mem, size_t size)
 *     cpdef copy_from_async(self, mem, size_t size, stream=?)             # <<<<<<<<<<<<<<
 *     cpdef copy_to_host(self, mem, size_t size)
 *     cpdef copy_to_host_async(self, mem, size_t size, stream=?)
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_async {
  int __pyx_n;
  PyObject *stream;
};

/* "memory.pxd":46
 *     cpdef copy_from_async(self, mem, size_t size, stream=?)
 *     cpdef copy_to_host(self, mem, size_t size)
 *     cpdef copy_to_host_async(self, mem, size_t size, stream=?)             # <<<<<<<<<<<<<<
 *     cpdef memset(self, int value, size_t size)
 *     cpdef memset_async(self, int value, size_t size, stream=?)
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_to_host_async {
  int __pyx_n;
  PyObject *stream;
};

/* "memory.pxd":48
 *     cpdef copy_to_host_async(self, mem, size_t size, stream=?)
 *     cpdef memset(self, int value, size_t size)
 *     cpdef memset_async(self, int value, size_t size, stream=?)             # <<<<<<<<<<<<<<
 * 
 * 
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_memset_async {
  int __pyx_n;
  PyObject *stream;
};

/* "memory.pxd":54
 * 
 * 
 * cpdef set_allocator(allocator=*)             # <<<<<<<<<<<<<<
 * cpdef get_allocator()
 * 
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_set_allocator {
  int __pyx_n;
  PyObject *allocator;
};

/* "memory.pxd":65
 * 
 *     cpdef MemoryPointer malloc(self, size_t size)
 *     cpdef free_all_blocks(self, stream=?)             # <<<<<<<<<<<<<<
 *     cpdef free_all_free(self)
 *     # they call device_pool so they may raise error.
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_10MemoryPool_free_all_blocks {
  int __pyx_n;
  PyObject *stream;
};

/* "memory.pxd":72
 *     cpdef size_t free_bytes(self)
 *     cpdef size_t total_bytes(self)
 *     cpdef set_limit(self, size=?, fraction=?)             # <<<<<<<<<<<<<<
 *     cpdef size_t get_limit(self)
 * 
*/
struct __pyx_opt_args_4cupy_4cuda_6memory_10MemoryPool_set_limit {
  int __pyx_n;
  PyObject *size;
  PyObject *fraction;
};
struct __pyx_opt_args_4cupy_4cuda_8function_8Function_linear_launch;

/* "cupy/cuda/function.pxd":15
 *         public intptr_t ptr
 * 
 *     cpdef linear_launch(self, size_t size, args, size_t shared_mem=*,             # <<<<<<<<<<<<<<
 *                         size_t block_max_size=*, stream=*,
 *                         bint enable_cooperative_groups=*)
*/
struct __pyx_opt_args_4cupy_4cuda_8function_8Function_linear_launch {
  int __pyx_n;
  size_t shared_mem;
  size_t block_max_size;
  PyObject *stream;
  int enable_cooperative_groups;
};
struct __pyx_t_4cupy_5_core_7_carray__CArray;
struct __pyx_t_4cupy_5_core_7_carray__CIndexer;

/* "cupy/_core/_carray.pxd":13
 * # this matches NPY_MAXDIMS
 * # Note: we make it an enum to work around cython/cython#4369
 * cdef enum: MAX_NDIM = 64             # <<<<<<<<<<<<<<
 * 
 * 
*/
enum  {
  __pyx_e_4cupy_5_core_7_carray_MAX_NDIM = 64
};

/* "cupy/_core/_carray.pxd":8
 * 
 * 
 * ctypedef vector.vector[Py_ssize_t] shape_t             # <<<<<<<<<<<<<<
 * ctypedef vector.vector[Py_ssize_t] strides_t
 * 
*/
typedef std::vector<Py_ssize_t>  __pyx_t_4cupy_5_core_7_carray_shape_t;

/* "cupy/_core/_carray.pxd":9
 * 
 * ctypedef vector.vector[Py_ssize_t] shape_t
 * ctypedef vector.vector[Py_ssize_t] strides_t             # <<<<<<<<<<<<<<
 * 
 * # this matches NPY_MAXDIMS
*/
typedef std::vector<Py_ssize_t>  __pyx_t_4cupy_5_core_7_carray_strides_t;

/* "cupy/_core/_carray.pxd":17
 * 
 * # This is now excluding shape/strides only for getting the compiler-time size
 * cdef struct _CArray:             # <<<<<<<<<<<<<<
 *     void* data
 *     Py_ssize_t size
*/
struct __pyx_t_4cupy_5_core_7_carray__CArray {
  void *data;
  Py_ssize_t size;
};

/* "cupy/_core/_carray.pxd":40
 * 
 * # This is now excluding shape/strides only for getting the compiler-time size
 * cdef struct _CIndexer:             # <<<<<<<<<<<<<<
 *     Py_ssize_t size
 * 
*/
struct __pyx_t_4cupy_5_core_7_carray__CIndexer {
  Py_ssize_t size;
};
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_tobytes;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_tofile;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base__astype;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_astype;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_byteswap;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_copy;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_view;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_flatten;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_ravel;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_squeeze;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_take;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_put;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_repeat;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_choose;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_sort;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argsort;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_partition;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argpartition;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_compress;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_diagonal;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_max;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argmax;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_min;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argmin;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_ptp;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_clip;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_round;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_trace;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_sum;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_cumsum;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_mean;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_var;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_std;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_prod;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_cumprod;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_all;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_any;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_get;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_set;
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_reduced_view;
struct __pyx_opt_args_4cupy_5_core_4core_ascontiguousarray;
struct __pyx_opt_args_4cupy_5_core_4core_asfortranarray;
struct __pyx_opt_args_4cupy_5_core_4core_compile_with_cache;
struct __pyx_opt_args_4cupy_5_core_4core_array;

/* "cupy/_core/core.pxd":31
 *     cpdef item(self)
 *     cpdef tolist(self)
 *     cpdef bytes tobytes(self, order=*)             # <<<<<<<<<<<<<<
 *     cpdef tofile(self, fid, sep=*, format=*)
 *     cpdef dump(self, file)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_tobytes {
  int __pyx_n;
  PyObject *order;
};

/* "cupy/_core/core.pxd":32
 *     cpdef tolist(self)
 *     cpdef bytes tobytes(self, order=*)
 *     cpdef tofile(self, fid, sep=*, format=*)             # <<<<<<<<<<<<<<
 *     cpdef dump(self, file)
 *     cpdef bytes dumps(self)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_tofile {
  int __pyx_n;
  PyObject *sep;
  PyObject *format;
};

/* "cupy/_core/core.pxd":35
 *     cpdef dump(self, file)
 *     cpdef bytes dumps(self)
 *     cpdef _ndarray_base _astype(             # <<<<<<<<<<<<<<
 *         self, dtype, order=*, casting=*, subok=*, copy=*)
 *     cpdef _ndarray_base astype(
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base__astype {
  int __pyx_n;
  PyObject *order;
  PyObject *casting;
  PyObject *subok;
  PyObject *copy;
};

/* "cupy/_core/core.pxd":37
 *     cpdef _ndarray_base _astype(
 *         self, dtype, order=*, casting=*, subok=*, copy=*)
 *     cpdef _ndarray_base astype(             # <<<<<<<<<<<<<<
 *         self, dtype, order=*, casting=*, subok=*, copy=*)
 *     cpdef _ndarray_base byteswap(self, inplace=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_astype {
  int __pyx_n;
  PyObject *order;
  PyObject *casting;
  PyObject *subok;
  PyObject *copy;
};

/* "cupy/_core/core.pxd":39
 *     cpdef _ndarray_base astype(
 *         self, dtype, order=*, casting=*, subok=*, copy=*)
 *     cpdef _ndarray_base byteswap(self, inplace=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base copy(self, order=*)
 *     cpdef _ndarray_base view(self, dtype=*, array_class=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_byteswap {
  int __pyx_n;
  PyObject *inplace;
};

/* "cupy/_core/core.pxd":40
 *         self, dtype, order=*, casting=*, subok=*, copy=*)
 *     cpdef _ndarray_base byteswap(self, inplace=*)
 *     cpdef _ndarray_base copy(self, order=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base view(self, dtype=*, array_class=*)
 *     cpdef fill(self, value)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_copy {
  int __pyx_n;
  PyObject *order;
};

/* "cupy/_core/core.pxd":41
 *     cpdef _ndarray_base byteswap(self, inplace=*)
 *     cpdef _ndarray_base copy(self, order=*)
 *     cpdef _ndarray_base view(self, dtype=*, array_class=*)             # <<<<<<<<<<<<<<
 *     cpdef fill(self, value)
 *     cpdef _ndarray_base swapaxes(self, Py_ssize_t axis1, Py_ssize_t axis2)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_view {
  int __pyx_n;
  PyObject *dtype;
  PyObject *array_class;
};

/* "cupy/_core/core.pxd":44
 *     cpdef fill(self, value)
 *     cpdef _ndarray_base swapaxes(self, Py_ssize_t axis1, Py_ssize_t axis2)
 *     cpdef _ndarray_base flatten(self, order=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base ravel(self, order=*)
 *     cpdef _ndarray_base squeeze(self, axis=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_flatten {
  int __pyx_n;
  PyObject *order;
};

/* "cupy/_core/core.pxd":45
 *     cpdef _ndarray_base swapaxes(self, Py_ssize_t axis1, Py_ssize_t axis2)
 *     cpdef _ndarray_base flatten(self, order=*)
 *     cpdef _ndarray_base ravel(self, order=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base squeeze(self, axis=*)
 *     cpdef _ndarray_base take(self, indices, axis=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_ravel {
  int __pyx_n;
  PyObject *order;
};

/* "cupy/_core/core.pxd":46
 *     cpdef _ndarray_base flatten(self, order=*)
 *     cpdef _ndarray_base ravel(self, order=*)
 *     cpdef _ndarray_base squeeze(self, axis=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base take(self, indices, axis=*, out=*)
 *     cpdef put(self, indices, values, mode=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_squeeze {
  int __pyx_n;
  PyObject *axis;
};

/* "cupy/_core/core.pxd":47
 *     cpdef _ndarray_base ravel(self, order=*)
 *     cpdef _ndarray_base squeeze(self, axis=*)
 *     cpdef _ndarray_base take(self, indices, axis=*, out=*)             # <<<<<<<<<<<<<<
 *     cpdef put(self, indices, values, mode=*)
 *     cpdef repeat(self, repeats, axis=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_take {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
};

/* "cupy/_core/core.pxd":48
 *     cpdef _ndarray_base squeeze(self, axis=*)
 *     cpdef _ndarray_base take(self, indices, axis=*, out=*)
 *     cpdef put(self, indices, values, mode=*)             # <<<<<<<<<<<<<<
 *     cpdef repeat(self, repeats, axis=*)
 *     cpdef choose(self, choices, out=*, mode=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_put {
  int __pyx_n;
  PyObject *mode;
};

/* "cupy/_core/core.pxd":49
 *     cpdef _ndarray_base take(self, indices, axis=*, out=*)
 *     cpdef put(self, indices, values, mode=*)
 *     cpdef repeat(self, repeats, axis=*)             # <<<<<<<<<<<<<<
 *     cpdef choose(self, choices, out=*, mode=*)
 *     cpdef sort(self, int axis=*, kind=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_repeat {
  int __pyx_n;
  PyObject *axis;
};

/* "cupy/_core/core.pxd":50
 *     cpdef put(self, indices, values, mode=*)
 *     cpdef repeat(self, repeats, axis=*)
 *     cpdef choose(self, choices, out=*, mode=*)             # <<<<<<<<<<<<<<
 *     cpdef sort(self, int axis=*, kind=*)
 *     cpdef _ndarray_base argsort(self, axis=*, kind=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_choose {
  int __pyx_n;
  PyObject *out;
  PyObject *mode;
};

/* "cupy/_core/core.pxd":51
 *     cpdef repeat(self, repeats, axis=*)
 *     cpdef choose(self, choices, out=*, mode=*)
 *     cpdef sort(self, int axis=*, kind=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base argsort(self, axis=*, kind=*)
 *     cpdef partition(self, kth, int axis=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_sort {
  int __pyx_n;
  int axis;
  PyObject *kind;
};

/* "cupy/_core/core.pxd":52
 *     cpdef choose(self, choices, out=*, mode=*)
 *     cpdef sort(self, int axis=*, kind=*)
 *     cpdef _ndarray_base argsort(self, axis=*, kind=*)             # <<<<<<<<<<<<<<
 *     cpdef partition(self, kth, int axis=*)
 *     cpdef _ndarray_base argpartition(self, kth, axis=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argsort {
  int __pyx_n;
  PyObject *axis;
  PyObject *kind;
};

/* "cupy/_core/core.pxd":53
 *     cpdef sort(self, int axis=*, kind=*)
 *     cpdef _ndarray_base argsort(self, axis=*, kind=*)
 *     cpdef partition(self, kth, int axis=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base argpartition(self, kth, axis=*)
 *     cpdef tuple nonzero(self)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_partition {
  int __pyx_n;
  int axis;
};

/* "cupy/_core/core.pxd":54
 *     cpdef _ndarray_base argsort(self, axis=*, kind=*)
 *     cpdef partition(self, kth, int axis=*)
 *     cpdef _ndarray_base argpartition(self, kth, axis=*)             # <<<<<<<<<<<<<<
 *     cpdef tuple nonzero(self)
 *     cpdef _ndarray_base compress(self, condition, axis=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argpartition {
  int __pyx_n;
  PyObject *axis;
};

/* "cupy/_core/core.pxd":56
 *     cpdef _ndarray_base argpartition(self, kth, axis=*)
 *     cpdef tuple nonzero(self)
 *     cpdef _ndarray_base compress(self, condition, axis=*, out=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base diagonal(self, offset=*, axis1=*, axis2=*)
 *     cpdef _ndarray_base max(self, axis=*, out=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_compress {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
};

/* "cupy/_core/core.pxd":57
 *     cpdef tuple nonzero(self)
 *     cpdef _ndarray_base compress(self, condition, axis=*, out=*)
 *     cpdef _ndarray_base diagonal(self, offset=*, axis1=*, axis2=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base max(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base argmax(self, axis=*, out=*, dtype=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_diagonal {
  int __pyx_n;
  PyObject *offset;
  PyObject *axis1;
  PyObject *axis2;
};

/* "cupy/_core/core.pxd":58
 *     cpdef _ndarray_base compress(self, condition, axis=*, out=*)
 *     cpdef _ndarray_base diagonal(self, offset=*, axis1=*, axis2=*)
 *     cpdef _ndarray_base max(self, axis=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base argmax(self, axis=*, out=*, dtype=*, keepdims=*)
 *     cpdef _ndarray_base min(self, axis=*, out=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_max {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":59
 *     cpdef _ndarray_base diagonal(self, offset=*, axis1=*, axis2=*)
 *     cpdef _ndarray_base max(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base argmax(self, axis=*, out=*, dtype=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base min(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base argmin(self, axis=*, out=*, dtype=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argmax {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *dtype;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":60
 *     cpdef _ndarray_base max(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base argmax(self, axis=*, out=*, dtype=*, keepdims=*)
 *     cpdef _ndarray_base min(self, axis=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base argmin(self, axis=*, out=*, dtype=*, keepdims=*)
 *     cpdef _ndarray_base ptp(self, axis=*, out=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_min {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":61
 *     cpdef _ndarray_base argmax(self, axis=*, out=*, dtype=*, keepdims=*)
 *     cpdef _ndarray_base min(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base argmin(self, axis=*, out=*, dtype=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base ptp(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base clip(self, min=*, max=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argmin {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *dtype;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":62
 *     cpdef _ndarray_base min(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base argmin(self, axis=*, out=*, dtype=*, keepdims=*)
 *     cpdef _ndarray_base ptp(self, axis=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base clip(self, min=*, max=*, out=*)
 *     cpdef _ndarray_base round(self, decimals=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_ptp {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":63
 *     cpdef _ndarray_base argmin(self, axis=*, out=*, dtype=*, keepdims=*)
 *     cpdef _ndarray_base ptp(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base clip(self, min=*, max=*, out=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base round(self, decimals=*, out=*)
 * 
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_clip {
  int __pyx_n;
  PyObject *min;
  PyObject *max;
  PyObject *out;
};

/* "cupy/_core/core.pxd":64
 *     cpdef _ndarray_base ptp(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base clip(self, min=*, max=*, out=*)
 *     cpdef _ndarray_base round(self, decimals=*, out=*)             # <<<<<<<<<<<<<<
 * 
 *     cpdef _ndarray_base trace(self, offset=*, axis1=*, axis2=*, dtype=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_round {
  int __pyx_n;
  PyObject *decimals;
  PyObject *out;
};

/* "cupy/_core/core.pxd":66
 *     cpdef _ndarray_base round(self, decimals=*, out=*)
 * 
 *     cpdef _ndarray_base trace(self, offset=*, axis1=*, axis2=*, dtype=*, out=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base sum(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base cumsum(self, axis=*, dtype=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_trace {
  int __pyx_n;
  PyObject *offset;
  PyObject *axis1;
  PyObject *axis2;
  PyObject *dtype;
  PyObject *out;
};

/* "cupy/_core/core.pxd":67
 * 
 *     cpdef _ndarray_base trace(self, offset=*, axis1=*, axis2=*, dtype=*, out=*)
 *     cpdef _ndarray_base sum(self, axis=*, dtype=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base cumsum(self, axis=*, dtype=*, out=*)
 *     cpdef _ndarray_base mean(self, axis=*, dtype=*, out=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_sum {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":68
 *     cpdef _ndarray_base trace(self, offset=*, axis1=*, axis2=*, dtype=*, out=*)
 *     cpdef _ndarray_base sum(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base cumsum(self, axis=*, dtype=*, out=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base mean(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base var(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_cumsum {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
};

/* "cupy/_core/core.pxd":69
 *     cpdef _ndarray_base sum(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base cumsum(self, axis=*, dtype=*, out=*)
 *     cpdef _ndarray_base mean(self, axis=*, dtype=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base var(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
 *     cpdef _ndarray_base std(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_mean {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":70
 *     cpdef _ndarray_base cumsum(self, axis=*, dtype=*, out=*)
 *     cpdef _ndarray_base mean(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base var(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base std(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
 *     cpdef _ndarray_base prod(self, axis=*, dtype=*, out=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_var {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
  PyObject *ddof;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":71
 *     cpdef _ndarray_base mean(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base var(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
 *     cpdef _ndarray_base std(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base prod(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base cumprod(self, axis=*, dtype=*, out=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_std {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
  PyObject *ddof;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":72
 *     cpdef _ndarray_base var(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
 *     cpdef _ndarray_base std(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
 *     cpdef _ndarray_base prod(self, axis=*, dtype=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base cumprod(self, axis=*, dtype=*, out=*)
 *     cpdef _ndarray_base _add_reduceat(self, indices, axis, dtype, out)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_prod {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":73
 *     cpdef _ndarray_base std(self, axis=*, dtype=*, out=*, ddof=*, keepdims=*)
 *     cpdef _ndarray_base prod(self, axis=*, dtype=*, out=*, keepdims=*)
 *     cpdef _ndarray_base cumprod(self, axis=*, dtype=*, out=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base _add_reduceat(self, indices, axis, dtype, out)
 *     cpdef _ndarray_base all(self, axis=*, out=*, keepdims=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_cumprod {
  int __pyx_n;
  PyObject *axis;
  PyObject *dtype;
  PyObject *out;
};

/* "cupy/_core/core.pxd":75
 *     cpdef _ndarray_base cumprod(self, axis=*, dtype=*, out=*)
 *     cpdef _ndarray_base _add_reduceat(self, indices, axis, dtype, out)
 *     cpdef _ndarray_base all(self, axis=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base any(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base conj(self)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_all {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":76
 *     cpdef _ndarray_base _add_reduceat(self, indices, axis, dtype, out)
 *     cpdef _ndarray_base all(self, axis=*, out=*, keepdims=*)
 *     cpdef _ndarray_base any(self, axis=*, out=*, keepdims=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base conj(self)
 *     cpdef _ndarray_base conjugate(self)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_any {
  int __pyx_n;
  PyObject *axis;
  PyObject *out;
  PyObject *keepdims;
};

/* "cupy/_core/core.pxd":79
 *     cpdef _ndarray_base conj(self)
 *     cpdef _ndarray_base conjugate(self)
 *     cpdef get(self, stream=*, order=*, out=*, blocking=*)             # <<<<<<<<<<<<<<
 *     cpdef set(self, arr, stream=*)
 *     cpdef _ndarray_base reduced_view(self, dtype=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_get {
  int __pyx_n;
  PyObject *stream;
  PyObject *order;
  PyObject *out;
  PyObject *blocking;
};

/* "cupy/_core/core.pxd":80
 *     cpdef _ndarray_base conjugate(self)
 *     cpdef get(self, stream=*, order=*, out=*, blocking=*)
 *     cpdef set(self, arr, stream=*)             # <<<<<<<<<<<<<<
 *     cpdef _ndarray_base reduced_view(self, dtype=*)
 *     cdef _update_c_contiguity(self)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_set {
  int __pyx_n;
  PyObject *stream;
};

/* "cupy/_core/core.pxd":81
 *     cpdef get(self, stream=*, order=*, out=*, blocking=*)
 *     cpdef set(self, arr, stream=*)
 *     cpdef _ndarray_base reduced_view(self, dtype=*)             # <<<<<<<<<<<<<<
 *     cdef _update_c_contiguity(self)
 *     cdef _update_f_contiguity(self)
*/
struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_reduced_view {
  int __pyx_n;
  PyObject *dtype;
};

/* "cupy/_core/core.pxd":101
 * cpdef _ndarray_base _internal_ascontiguousarray(_ndarray_base a)
 * cpdef _ndarray_base _internal_asfortranarray(_ndarray_base a)
 * cpdef _ndarray_base ascontiguousarray(_ndarray_base a, dtype=*)             # <<<<<<<<<<<<<<
 * cpdef _ndarray_base asfortranarray(_ndarray_base a, dtype=*)
 * 
*/
struct __pyx_opt_args_4cupy_5_core_4core_ascontiguousarray {
  int __pyx_n;
  PyObject *dtype;
};

/* "cupy/_core/core.pxd":102
 * cpdef _ndarray_base _internal_asfortranarray(_ndarray_base a)
 * cpdef _ndarray_base ascontiguousarray(_ndarray_base a, dtype=*)
 * cpdef _ndarray_base asfortranarray(_ndarray_base a, dtype=*)             # <<<<<<<<<<<<<<
 * 
 * cpdef Module compile_with_cache(str source, tuple options=*, arch=*,
*/
struct __pyx_opt_args_4cupy_5_core_4core_asfortranarray {
  int __pyx_n;
  PyObject *dtype;
};

/* "cupy/_core/core.pxd":104
 * cpdef _ndarray_base asfortranarray(_ndarray_base a, dtype=*)
 * 
 * cpdef Module compile_with_cache(str source, tuple options=*, arch=*,             # <<<<<<<<<<<<<<
 *                                 prepend_cupy_headers=*,
 *                                 backend=*, translate_cucomplex=*,
*/
struct __pyx_opt_args_4cupy_5_core_4core_compile_with_cache {
  int __pyx_n;
  PyObject *options;
  PyObject *arch;
  PyObject *prepend_cupy_headers;
  PyObject *backend;
  PyObject *translate_cucomplex;
  PyObject *enable_cooperative_groups;
  PyObject *name_expressions;
  PyObject *log_stream;
  int jitify;
};

/* "cupy/_core/core.pxd":114
 * # TODO(niboshi): Move to _routines_creation.pyx
 * 
 * cpdef _ndarray_base array(             # <<<<<<<<<<<<<<
 *     obj, dtype=*, copy=*, order=*, bint subok=*, Py_ssize_t ndmin=*,
 *     bint blocking=*)
*/
struct __pyx_opt_args_4cupy_5_core_4core_array {
  int __pyx_n;
  PyObject *dtype;
  PyObject *copy;
  PyObject *order;
  int subok;
  Py_ssize_t ndmin;
  int blocking;
};
struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos;

/* "cupy/fft/_callback.pyx":348
 *         readonly str identity
 * 
 *     cdef set_caller_infos(self,             # <<<<<<<<<<<<<<
 *                           MemoryPointer cb_load_data=None,
 *                           MemoryPointer cb_store_data=None):
*/
struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos {
  int __pyx_n;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *cb_load_data;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *cb_store_data;
};

/* "runtime.pxd":8
 * ###############################################################################
 * 
 * cdef class PointerAttributes:             # <<<<<<<<<<<<<<
 *     cdef:
 *         readonly int device
*/
struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes {
  PyObject_HEAD
  int device;
  intptr_t devicePointer;
  intptr_t hostPointer;
  int type;
};


/* "runtime.pxd":15
 *         readonly int type
 * 
 * cdef class MemPoolProps:             # <<<<<<<<<<<<<<
 *     # flatten the struct & list meaningful members
 *     cdef:
*/
struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps {
  PyObject_HEAD
  int allocType;
  int handleType;
  int locationType;
  int devId;
};


/* "device.pxd":13
 * cdef bint _enable_peer_access(int device, int peer) except -1
 * 
 * cdef class Handle:             # <<<<<<<<<<<<<<
 *     cdef:
 *         public size_t handle
*/
struct __pyx_obj_4cupy_4cuda_6device_Handle {
  PyObject_HEAD
  size_t handle;
  PyObject *_destroy_func;
};


/* "device.pxd":18
 *         object _destroy_func
 * 
 * cdef class Device:             # <<<<<<<<<<<<<<
 *     cdef:
 *         public int id
*/
struct __pyx_obj_4cupy_4cuda_6device_Device {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_6device_Device *__pyx_vtab;
  int id;
  PyObject *_device_stack;
};


/* "memory.pxd":10
 * 
 * 
 * @cython.no_gc             # <<<<<<<<<<<<<<
 * cdef class BaseMemory:
 * 
*/
struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory {
  PyObject_HEAD
  intptr_t ptr;
  size_t size;
  int device_id;
};


/* "memory.pxd":19
 * 
 * 
 * @cython.no_gc             # <<<<<<<<<<<<<<
 * cdef class SystemMemory(BaseMemory):
 * 
*/
struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory {
  struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory __pyx_base;
  struct __pyx_vtabstruct_4cupy_4cuda_6memory_SystemMemory *__pyx_vtab;
  PyObject *_owner;
};


/* "memory.pxd":29
 * 
 * 
 * cdef class MemoryPointer:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPointer *__pyx_vtab;
  intptr_t ptr;
  int device_id;
  struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory *mem;
};


/* "memory.pxd":58
 * 
 * 
 * cdef class MemoryPool:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPool *__pyx_vtab;
  PyObject *_pools;
  PyObject *_allocator;
};


/* "memory.pxd":89
 * 
 * 
 * @cython.no_gc             # <<<<<<<<<<<<<<
 * cdef class CFunctionAllocatorMemory(BaseMemory):
 * 
*/
struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory {
  struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory __pyx_base;
  intptr_t _param;
  intptr_t _free_func;
};


/* "memory.pxd":97
 * 
 * 
 * cdef class CFunctionAllocator:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_6memory_CFunctionAllocator *__pyx_vtab;
  intptr_t _param;
  intptr_t _malloc_func;
  intptr_t _free_func;
  PyObject *_owner;
};


/* "memory.pxd":108
 * 
 * 
 * cdef class PythonFunctionAllocatorMemory(BaseMemory):             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory {
  struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory __pyx_base;
  PyObject *_free_func;
};


/* "memory.pxd":114
 * 
 * 
 * cdef class PythonFunctionAllocator:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_6memory_PythonFunctionAllocator *__pyx_vtab;
  PyObject *_malloc_func;
  PyObject *_free_func;
};


/* "cupy/cuda/function.pxd":4
 * 
 * 
 * cdef class CPointer:             # <<<<<<<<<<<<<<
 *     cdef:
 *         readonly intptr_t ptr
*/
struct __pyx_obj_4cupy_4cuda_8function_CPointer {
  PyObject_HEAD
  intptr_t ptr;
};


/* "cupy/cuda/function.pxd":9
 * 
 * 
 * cdef class Function:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_8function_Function {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_8function_Function *__pyx_vtab;
  struct __pyx_obj_4cupy_4cuda_8function_Module *module;
  intptr_t ptr;
};


/* "cupy/cuda/function.pxd":20
 * 
 * 
 * cdef class Module:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_8function_Module {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_8function_Module *__pyx_vtab;
  intptr_t ptr;
  PyObject *mapping;
};


/* "cupy/cuda/function.pxd":34
 * 
 * 
 * cdef class LinkState:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_4cuda_8function_LinkState {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_4cuda_8function_LinkState *__pyx_vtab;
  intptr_t ptr;
};


/* "cupy/_core/_carray.pxd":22
 * 
 * 
 * @cython.final             # <<<<<<<<<<<<<<
 * cdef class mdspan(function.CPointer):
 * 
*/
struct __pyx_obj_4cupy_5_core_7_carray_mdspan {
  struct __pyx_obj_4cupy_4cuda_8function_CPointer __pyx_base;
  struct __pyx_vtabstruct_4cupy_5_core_7_carray_mdspan *__pyx_vtab;
};


/* "cupy/_core/_carray.pxd":31
 * 
 * 
 * @cython.final             # <<<<<<<<<<<<<<
 * cdef class CArray(function.CPointer):
 * 
*/
struct __pyx_obj_4cupy_5_core_7_carray_CArray {
  struct __pyx_obj_4cupy_4cuda_8function_CPointer __pyx_base;
  struct __pyx_vtabstruct_4cupy_5_core_7_carray_CArray *__pyx_vtab;
};


/* "cupy/_core/_carray.pxd":44
 * 
 * 
 * cdef class CIndexer(function.CPointer):             # <<<<<<<<<<<<<<
 * 
 *     cdef void init(self, Py_ssize_t size, const shape_t &shape) except*
*/
struct __pyx_obj_4cupy_5_core_7_carray_CIndexer {
  struct __pyx_obj_4cupy_4cuda_8function_CPointer __pyx_base;
  struct __pyx_vtabstruct_4cupy_5_core_7_carray_CIndexer *__pyx_vtab;
};


/* "cupy/_core/_carray.pxd":49
 * 
 * 
 * cdef class Indexer:             # <<<<<<<<<<<<<<
 *     cdef:
 *         readonly Py_ssize_t size
*/
struct __pyx_obj_4cupy_5_core_7_carray_Indexer {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_5_core_7_carray_Indexer *__pyx_vtab;
  Py_ssize_t size;
  __pyx_t_4cupy_5_core_7_carray_shape_t shape;
  int _index_32_bits;
};


/* "cupy/_core/core.pxd":10
 * 
 * 
 * cdef class _ndarray_base:             # <<<<<<<<<<<<<<
 *     cdef:
 *         object __weakref__
*/
struct __pyx_obj_4cupy_5_core_4core__ndarray_base {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_5_core_4core__ndarray_base *__pyx_vtab;
  PyObject *__weakref__;
  Py_ssize_t size;
  __pyx_t_4cupy_5_core_7_carray_shape_t _shape;
  __pyx_t_4cupy_5_core_7_carray_strides_t _strides;
  int _c_contiguous;
  int _f_contiguous;
  int _index_32_bits;
  PyObject *dtype;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *data;
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *base;
};


/* "cupy/fft/_callback.pyx":53
 * 
 * 
 * cdef class _ThreadLocal:             # <<<<<<<<<<<<<<
 *     cdef _CallbackManager _current_cufft_callback
 * 
*/
struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_3fft_9_callback__ThreadLocal *__pyx_vtab;
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *_current_cufft_callback;
};


/* "cupy/fft/_callback.pyx":341
 * 
 * 
 * cdef class _CallbackManager:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager {
  PyObject_HEAD
  struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager *__pyx_vtab;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *cb_load_data;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *cb_store_data;
  PyObject *identity;
};


/* "cupy/fft/_callback.pyx":369
 * 
 * 
 * cdef class _LegacyCallbackManager(_CallbackManager):             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager {
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager __pyx_base;
  PyObject *cb_load;
  PyObject *cb_store;
  PyObject *mod;
};


/* "cupy/fft/_callback.pyx":536
 * 
 * 
 * cdef class _JITCallbackManager(_CallbackManager):             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/
struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager {
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager __pyx_base;
  PyObject *cb_load;
  PyObject *cb_store;
  PyObject *cb_load_name;
  PyObject *cb_store_name;
  PyObject *cb_load_lto;
  PyObject *cb_store_lto;
};


/* "cupy/fft/_callback.pyx":698
 * 
 * 
 * cdef class set_cufft_callbacks:             # <<<<<<<<<<<<<<
 *     """A context manager for setting up load and/or store callbacks.
 *     Any FFT calls living in this context will have callbacks set up.
*/
struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks {
  PyObject_HEAD
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *mgr;
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *mgr_prev;
};



/* "device.pxd":18
 *         object _destroy_func
 * 
 * cdef class Device:             # <<<<<<<<<<<<<<
 *     cdef:
 *         public int id
*/

struct __pyx_vtabstruct_4cupy_4cuda_6device_Device {
  PyObject *(*use)(struct __pyx_obj_4cupy_4cuda_6device_Device *, int __pyx_skip_dispatch);
  PyObject *(*synchronize)(struct __pyx_obj_4cupy_4cuda_6device_Device *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_4cuda_6device_Device *__pyx_vtabptr_4cupy_4cuda_6device_Device;


/* "memory.pxd":19
 * 
 * 
 * @cython.no_gc             # <<<<<<<<<<<<<<
 * cdef class SystemMemory(BaseMemory):
 * 
*/

struct __pyx_vtabstruct_4cupy_4cuda_6memory_SystemMemory {
  PyObject *(*from_external)(intptr_t, size_t, PyObject *);
};
static struct __pyx_vtabstruct_4cupy_4cuda_6memory_SystemMemory *__pyx_vtabptr_4cupy_4cuda_6memory_SystemMemory;


/* "memory.pxd":29
 * 
 * 
 * cdef class MemoryPointer:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPointer {
  PyObject *(*_init)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory *, ptrdiff_t);
  PyObject *(*copy_from_device)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, size_t, int __pyx_skip_dispatch);
  PyObject *(*copy_from_device_async)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, size_t, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_device_async *__pyx_optional_args);
  PyObject *(*copy_from_host)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, PyObject *, size_t, int __pyx_skip_dispatch);
  PyObject *(*copy_from_host_async)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, PyObject *, size_t, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_host_async *__pyx_optional_args);
  PyObject *(*copy_from)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, PyObject *, size_t, int __pyx_skip_dispatch);
  PyObject *(*copy_from_async)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, PyObject *, size_t, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_from_async *__pyx_optional_args);
  PyObject *(*copy_to_host)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, PyObject *, size_t, int __pyx_skip_dispatch);
  PyObject *(*copy_to_host_async)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, PyObject *, size_t, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_copy_to_host_async *__pyx_optional_args);
  PyObject *(*memset)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, int, size_t, int __pyx_skip_dispatch);
  PyObject *(*memset_async)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, int, size_t, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_13MemoryPointer_memset_async *__pyx_optional_args);
};
static struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPointer *__pyx_vtabptr_4cupy_4cuda_6memory_MemoryPointer;


/* "memory.pxd":58
 * 
 * 
 * cdef class MemoryPool:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPool {
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *(*malloc)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, size_t, int __pyx_skip_dispatch);
  PyObject *(*free_all_blocks)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_10MemoryPool_free_all_blocks *__pyx_optional_args);
  PyObject *(*free_all_free)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch);
  size_t (*n_free_blocks)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch);
  size_t (*used_bytes)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch);
  size_t (*free_bytes)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch);
  size_t (*total_bytes)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch);
  PyObject *(*set_limit)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_6memory_10MemoryPool_set_limit *__pyx_optional_args);
  size_t (*get_limit)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *, int __pyx_skip_dispatch);
  PyObject *(*_ensure_pools_and_return_device_pool)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *);
  PyObject *(*device_pool)(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *);
};
static struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPool *__pyx_vtabptr_4cupy_4cuda_6memory_MemoryPool;
static CYTHON_INLINE PyObject *__pyx_f_4cupy_4cuda_6memory_10MemoryPool_device_pool(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *);


/* "memory.pxd":97
 * 
 * 
 * cdef class CFunctionAllocator:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_6memory_CFunctionAllocator {
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *(*malloc)(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator *, size_t, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_4cuda_6memory_CFunctionAllocator *__pyx_vtabptr_4cupy_4cuda_6memory_CFunctionAllocator;


/* "memory.pxd":114
 * 
 * 
 * cdef class PythonFunctionAllocator:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_6memory_PythonFunctionAllocator {
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *(*malloc)(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator *, size_t, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_4cuda_6memory_PythonFunctionAllocator *__pyx_vtabptr_4cupy_4cuda_6memory_PythonFunctionAllocator;


/* "cupy/cuda/function.pxd":9
 * 
 * 
 * cdef class Function:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_8function_Function {
  PyObject *(*linear_launch)(struct __pyx_obj_4cupy_4cuda_8function_Function *, size_t, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_4cuda_8function_8Function_linear_launch *__pyx_optional_args);
};
static struct __pyx_vtabstruct_4cupy_4cuda_8function_Function *__pyx_vtabptr_4cupy_4cuda_8function_Function;


/* "cupy/cuda/function.pxd":20
 * 
 * 
 * cdef class Module:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_8function_Module {
  PyObject *(*load_file)(struct __pyx_obj_4cupy_4cuda_8function_Module *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*load)(struct __pyx_obj_4cupy_4cuda_8function_Module *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*get_global_var)(struct __pyx_obj_4cupy_4cuda_8function_Module *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*get_function)(struct __pyx_obj_4cupy_4cuda_8function_Module *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*_set_mapping)(struct __pyx_obj_4cupy_4cuda_8function_Module *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*_enumerate_and_build_mapping)(struct __pyx_obj_4cupy_4cuda_8function_Module *, PyObject *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_4cuda_8function_Module *__pyx_vtabptr_4cupy_4cuda_8function_Module;


/* "cupy/cuda/function.pxd":34
 * 
 * 
 * cdef class LinkState:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_4cuda_8function_LinkState {
  PyObject *(*add_ptr_data)(struct __pyx_obj_4cupy_4cuda_8function_LinkState *, PyObject *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*add_ptr_file)(struct __pyx_obj_4cupy_4cuda_8function_LinkState *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*complete)(struct __pyx_obj_4cupy_4cuda_8function_LinkState *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_4cuda_8function_LinkState *__pyx_vtabptr_4cupy_4cuda_8function_LinkState;


/* "cupy/_core/_carray.pxd":22
 * 
 * 
 * @cython.final             # <<<<<<<<<<<<<<
 * cdef class mdspan(function.CPointer):
 * 
*/

struct __pyx_vtabstruct_4cupy_5_core_7_carray_mdspan {
  int (*init)(struct __pyx_obj_4cupy_5_core_7_carray_mdspan *, void *, int, __pyx_t_4cupy_5_core_7_carray_shape_t const &, __pyx_t_4cupy_5_core_7_carray_strides_t const &, int, int);
};
static struct __pyx_vtabstruct_4cupy_5_core_7_carray_mdspan *__pyx_vtabptr_4cupy_5_core_7_carray_mdspan;


/* "cupy/_core/_carray.pxd":31
 * 
 * 
 * @cython.final             # <<<<<<<<<<<<<<
 * cdef class CArray(function.CPointer):
 * 
*/

struct __pyx_vtabstruct_4cupy_5_core_7_carray_CArray {
  void (*init)(struct __pyx_obj_4cupy_5_core_7_carray_CArray *, void *, Py_ssize_t, __pyx_t_4cupy_5_core_7_carray_shape_t const &, __pyx_t_4cupy_5_core_7_carray_strides_t const &);
};
static struct __pyx_vtabstruct_4cupy_5_core_7_carray_CArray *__pyx_vtabptr_4cupy_5_core_7_carray_CArray;


/* "cupy/_core/_carray.pxd":44
 * 
 * 
 * cdef class CIndexer(function.CPointer):             # <<<<<<<<<<<<<<
 * 
 *     cdef void init(self, Py_ssize_t size, const shape_t &shape) except*
*/

struct __pyx_vtabstruct_4cupy_5_core_7_carray_CIndexer {
  void (*init)(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer *, Py_ssize_t, __pyx_t_4cupy_5_core_7_carray_shape_t const &);
};
static struct __pyx_vtabstruct_4cupy_5_core_7_carray_CIndexer *__pyx_vtabptr_4cupy_5_core_7_carray_CIndexer;


/* "cupy/_core/_carray.pxd":49
 * 
 * 
 * cdef class Indexer:             # <<<<<<<<<<<<<<
 *     cdef:
 *         readonly Py_ssize_t size
*/

struct __pyx_vtabstruct_4cupy_5_core_7_carray_Indexer {
  void (*init)(struct __pyx_obj_4cupy_5_core_7_carray_Indexer *, __pyx_t_4cupy_5_core_7_carray_shape_t const &);
  struct __pyx_obj_4cupy_4cuda_8function_CPointer *(*get_pointer)(struct __pyx_obj_4cupy_5_core_7_carray_Indexer *);
};
static struct __pyx_vtabstruct_4cupy_5_core_7_carray_Indexer *__pyx_vtabptr_4cupy_5_core_7_carray_Indexer;


/* "cupy/_core/core.pxd":10
 * 
 * 
 * cdef class _ndarray_base:             # <<<<<<<<<<<<<<
 *     cdef:
 *         object __weakref__
*/

struct __pyx_vtabstruct_4cupy_5_core_4core__ndarray_base {
  PyObject *(*_init_fast)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, __pyx_t_4cupy_5_core_7_carray_shape_t const &, PyObject *, int);
  int (*is_host_accessible)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *);
  PyObject *(*item)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
  PyObject *(*tolist)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
  PyObject *(*tobytes)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_tobytes *__pyx_optional_args);
  PyObject *(*tofile)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_tofile *__pyx_optional_args);
  PyObject *(*dump)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*dumps)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*_astype)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base__astype *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*astype)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_astype *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*byteswap)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_byteswap *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*copy)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_copy *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*view)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_view *__pyx_optional_args);
  PyObject *(*fill)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*swapaxes)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, Py_ssize_t, Py_ssize_t, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*flatten)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_flatten *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*ravel)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_ravel *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*squeeze)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_squeeze *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*take)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_take *__pyx_optional_args);
  PyObject *(*put)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_put *__pyx_optional_args);
  PyObject *(*repeat)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_repeat *__pyx_optional_args);
  PyObject *(*choose)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_choose *__pyx_optional_args);
  PyObject *(*sort)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_sort *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*argsort)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argsort *__pyx_optional_args);
  PyObject *(*partition)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_partition *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*argpartition)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argpartition *__pyx_optional_args);
  PyObject *(*nonzero)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*compress)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_compress *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*diagonal)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_diagonal *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*max)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_max *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*argmax)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argmax *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*min)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_min *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*argmin)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_argmin *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*ptp)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_ptp *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*clip)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_clip *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*round)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_round *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*trace)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_trace *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*sum)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_sum *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*cumsum)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_cumsum *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*mean)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_mean *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*var)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_var *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*std)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_std *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*prod)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_prod *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*cumprod)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_cumprod *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*_add_reduceat)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, PyObject *, PyObject *, PyObject *, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*all)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_all *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*any)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_any *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*conj)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*conjugate)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
  PyObject *(*get)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_get *__pyx_optional_args);
  PyObject *(*set)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_set *__pyx_optional_args);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*reduced_view)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch, struct __pyx_opt_args_4cupy_5_core_4core_13_ndarray_base_reduced_view *__pyx_optional_args);
  PyObject *(*_update_c_contiguity)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *);
  PyObject *(*_update_f_contiguity)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *);
  PyObject *(*_update_contiguity)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *);
  PyObject *(*_set_shape_and_strides)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, __pyx_t_4cupy_5_core_7_carray_shape_t const &, __pyx_t_4cupy_5_core_7_carray_strides_t const &, int, int, int __pyx_skip_dispatch);
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*_view)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, PyObject *, __pyx_t_4cupy_5_core_7_carray_shape_t const &, __pyx_t_4cupy_5_core_7_carray_strides_t const &, int, int, PyObject *);
  PyObject *(*_set_contiguous_strides)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, Py_ssize_t, int);
  struct __pyx_obj_4cupy_4cuda_8function_CPointer *(*get_pointer)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *);
  PyObject *(*toDlpack)(struct __pyx_obj_4cupy_5_core_4core__ndarray_base *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_5_core_4core__ndarray_base *__pyx_vtabptr_4cupy_5_core_4core__ndarray_base;


/* "cupy/fft/_callback.pyx":53
 * 
 * 
 * cdef class _ThreadLocal:             # <<<<<<<<<<<<<<
 *     cdef _CallbackManager _current_cufft_callback
 * 
*/

struct __pyx_vtabstruct_4cupy_3fft_9_callback__ThreadLocal {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *(*get)(void);
};
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__ThreadLocal *__pyx_vtabptr_4cupy_3fft_9_callback__ThreadLocal;


/* "cupy/fft/_callback.pyx":341
 * 
 * 
 * cdef class _CallbackManager:             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager {
  PyObject *(*set_caller_infos)(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *, struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos *__pyx_optional_args);
};
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager *__pyx_vtabptr_4cupy_3fft_9_callback__CallbackManager;


/* "cupy/fft/_callback.pyx":369
 * 
 * 
 * cdef class _LegacyCallbackManager(_CallbackManager):             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_3fft_9_callback__LegacyCallbackManager {
  struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager __pyx_base;
  PyObject *(*create_plan)(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *, PyObject *, int __pyx_skip_dispatch);
  PyObject *(*set_callbacks)(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *, PyObject *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_vtabptr_4cupy_3fft_9_callback__LegacyCallbackManager;


/* "cupy/fft/_callback.pyx":536
 * 
 * 
 * cdef class _JITCallbackManager(_CallbackManager):             # <<<<<<<<<<<<<<
 * 
 *     cdef:
*/

struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager {
  struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager __pyx_base;
  PyObject *(*_get_cuda_include)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *);
  PyObject *(*_sanity_checks)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *, PyObject *, PyObject *, PyObject *);
  PyObject *(*compile_lto)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *, PyObject *);
  PyObject *(*create_plan)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, intptr_t, PyObject *, int __pyx_skip_dispatch);
  intptr_t (*set_callbacks)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *, int __pyx_skip_dispatch);
};
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager *__pyx_vtabptr_4cupy_3fft_9_callback__JITCallbackManager;
/* #### Code section: utility_code_proto ### */

/* --- Runtime support code (head) --- */
/* Refnanny.proto */
#ifndef CYTHON_REFNANNY
  #define CYTHON_REFNANNY 0
#endif
#if CYTHON_REFNANNY
  typedef struct {
    void (*INCREF)(void*, PyObject*, Py_ssize_t);
    void (*DECREF)(void*, PyObject*, Py_ssize_t);
    void (*GOTREF)(void*, PyObject*, Py_ssize_t);
    void (*GIVEREF)(void*, PyObject*, Py_ssize_t);
    void* (*SetupContext)(const char*, Py_ssize_t, const char*);
    void (*FinishContext)(void**);
  } __Pyx_RefNannyAPIStruct;
  static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL;
  static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname);
  #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL;
  #define __Pyx_RefNannySetupContext(name, acquire_gil)\
          if (acquire_gil) {\
              PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\
              __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\
              PyGILState_Release(__pyx_gilstate_save);\
          } else {\
              __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\
          }
  #define __Pyx_RefNannyFinishContextNogil() {\
              PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\
              __Pyx_RefNannyFinishContext();\
              PyGILState_Release(__pyx_gilstate_save);\
          }
  #define __Pyx_RefNannyFinishContextNogil() {\
              PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\
              __Pyx_RefNannyFinishContext();\
              PyGILState_Release(__pyx_gilstate_save);\
          }
  #define __Pyx_RefNannyFinishContext()\
          __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
  #define __Pyx_INCREF(r)  __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
  #define __Pyx_DECREF(r)  __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
  #define __Pyx_GOTREF(r)  __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
  #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
  #define __Pyx_XINCREF(r)  do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0)
  #define __Pyx_XDECREF(r)  do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0)
  #define __Pyx_XGOTREF(r)  do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0)
  #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0)
#else
  #define __Pyx_RefNannyDeclarations
  #define __Pyx_RefNannySetupContext(name, acquire_gil)
  #define __Pyx_RefNannyFinishContextNogil()
  #define __Pyx_RefNannyFinishContext()
  #define __Pyx_INCREF(r) Py_INCREF(r)
  #define __Pyx_DECREF(r) Py_DECREF(r)
  #define __Pyx_GOTREF(r)
  #define __Pyx_GIVEREF(r)
  #define __Pyx_XINCREF(r) Py_XINCREF(r)
  #define __Pyx_XDECREF(r) Py_XDECREF(r)
  #define __Pyx_XGOTREF(r)
  #define __Pyx_XGIVEREF(r)
#endif
#define __Pyx_Py_XDECREF_SET(r, v) do {\
        PyObject *tmp = (PyObject *) r;\
        r = v; Py_XDECREF(tmp);\
    } while (0)
#define __Pyx_XDECREF_SET(r, v) do {\
        PyObject *tmp = (PyObject *) r;\
        r = v; __Pyx_XDECREF(tmp);\
    } while (0)
#define __Pyx_DECREF_SET(r, v) do {\
        PyObject *tmp = (PyObject *) r;\
        r = v; __Pyx_DECREF(tmp);\
    } while (0)
#define __Pyx_CLEAR(r)    do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0)
#define __Pyx_XCLEAR(r)   do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0)

/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */
#if CYTHON_FAST_THREAD_STATE
#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err)
static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err);
#else
#define __Pyx_PyErr_ExceptionMatches(err)  PyErr_ExceptionMatches(err)
#endif

/* PyThreadStateGet.proto (used by PyErrFetchRestore) */
#if CYTHON_FAST_THREAD_STATE
#define __Pyx_PyThreadState_declare  PyThreadState *__pyx_tstate;
#define __Pyx_PyThreadState_assign  __pyx_tstate = __Pyx_PyThreadState_Current;
#if PY_VERSION_HEX >= 0x030C00A6
#define __Pyx_PyErr_Occurred()  (__pyx_tstate->current_exception != NULL)
#define __Pyx_PyErr_CurrentExceptionType()  (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL)
#else
#define __Pyx_PyErr_Occurred()  (__pyx_tstate->curexc_type != NULL)
#define __Pyx_PyErr_CurrentExceptionType()  (__pyx_tstate->curexc_type)
#endif
#else
#define __Pyx_PyThreadState_declare
#define __Pyx_PyThreadState_assign
#define __Pyx_PyErr_Occurred()  (PyErr_Occurred() != NULL)
#define __Pyx_PyErr_CurrentExceptionType()  PyErr_Occurred()
#endif

/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */
#if CYTHON_FAST_THREAD_STATE
#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL)
#define __Pyx_ErrRestoreWithState(type, value, tb)  __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb)
#define __Pyx_ErrFetchWithState(type, value, tb)    __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb)
#define __Pyx_ErrRestore(type, value, tb)  __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb)
#define __Pyx_ErrFetch(type, value, tb)    __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb)
static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb);
static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6
#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL))
#else
#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc)
#endif
#else
#define __Pyx_PyErr_Clear() PyErr_Clear()
#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc)
#define __Pyx_ErrRestoreWithState(type, value, tb)  PyErr_Restore(type, value, tb)
#define __Pyx_ErrFetchWithState(type, value, tb)  PyErr_Fetch(type, value, tb)
#define __Pyx_ErrRestoreInState(tstate, type, value, tb)  PyErr_Restore(type, value, tb)
#define __Pyx_ErrFetchInState(tstate, type, value, tb)  PyErr_Fetch(type, value, tb)
#define __Pyx_ErrRestore(type, value, tb)  PyErr_Restore(type, value, tb)
#define __Pyx_ErrFetch(type, value, tb)  PyErr_Fetch(type, value, tb)
#endif

/* PyObjectGetAttrStr.proto (used by PyObjectGetAttrStrNoError) */
#if CYTHON_USE_TYPE_SLOTS
static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name);
#else
#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n)
#endif

/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name);

/* GetBuiltinName.proto */
static PyObject *__Pyx_GetBuiltinName(PyObject *name);

/* GetItemInt.proto */
#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\
    (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
    __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\
    (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\
               __Pyx_GetItemInt_Generic(o, to_py_func(i))))
#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\
    (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
    __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\
    (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL))
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i,
                                                              int wraparound, int boundscheck, int unsafe_shared);
#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\
    (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
    __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\
    (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL))
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i,
                                                              int wraparound, int boundscheck, int unsafe_shared);
static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j);
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i,
                                                     int is_list, int wraparound, int boundscheck, int unsafe_shared);

/* HasAttr.proto */
#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
#define __Pyx_HasAttr(o, n)  PyObject_HasAttrWithError(o, n)
#else
static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *);
#endif

/* PyObjectCall.proto (used by PyObjectFastCall) */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw);
#else
#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw)
#endif

/* PyObjectCallMethO.proto (used by PyObjectFastCall) */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg);
#endif

/* PyObjectFastCall.proto (used by PyObjectFastCallMethod) */
#define __Pyx_PyObject_FastCall(func, args, nargs)  __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL)
static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs);

/* PyObjectFastCallMethod.proto */
#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000
#define __Pyx_PyObject_FastCallMethod(name, args, nargsf) PyObject_VectorcallMethod(name, args, nargsf, NULL)
#else
static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf);
#endif

/* ExtTypeTest.proto */
static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type);

/* PyTypeError_Check.proto */
#define __Pyx_PyExc_TypeError_Check(obj)  __Pyx_TypeCheck(obj, PyExc_TypeError)

/* PyObjectFormatSimple.proto */
#if CYTHON_COMPILING_IN_PYPY
    #define __Pyx_PyObject_FormatSimple(s, f) (\
        likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\
        PyObject_Format(s, f))
#elif CYTHON_USE_TYPE_SLOTS
    #define __Pyx_PyObject_FormatSimple(s, f) (\
        likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\
        likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) :\
        likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) :\
        PyObject_Format(s, f))
#else
    #define __Pyx_PyObject_FormatSimple(s, f) (\
        likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\
        PyObject_Format(s, f))
#endif

/* IncludeStringH.proto (used by JoinPyUnicode) */
#include <string.h>

/* JoinPyUnicode.export */
static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength,
                                      Py_UCS4 max_char);

/* RaiseException.export */
static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause);

/* TupleAndListFromArray.proto (used by fastcall) */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n);
#endif
#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL
static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n);
#endif

/* BytesEquals.proto (used by UnicodeEquals) */
static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals);

/* UnicodeEquals.proto (used by fastcall) */
static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals);

/* fastcall.proto */
#if CYTHON_AVOID_BORROWED_REFS
    #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i)
#elif CYTHON_ASSUME_SAFE_MACROS
    #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i))
#else
    #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i))
#endif
#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds)
#define __Pyx_KwValues_VARARGS(args, nargs) NULL
#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s)
#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw)
#if CYTHON_METH_FASTCALL
    #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i])
    #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds)
    #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs))
    static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s);
  #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API
    CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues);
  #else
    #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw)
  #endif
#else
    #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS
    #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS
    #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS
    #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS
    #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS
#endif
#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop)
#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start)
#else
#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop)
#endif

/* RaiseArgTupleInvalid.proto */
static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact,
    Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found);

/* py_dict_items.proto (used by OwnedDictNext) */
static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d);

/* CallCFunction.proto (used by CallUnboundCMethod0) */
#define __Pyx_CallCFunction(cfunc, self, args)\
    ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args)
#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\
    ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs)
#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\
    ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs)
#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\
    ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames)

/* PyObjectCallOneArg.proto (used by CallUnboundCMethod0) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg);

/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod0) */
typedef struct {
    PyObject *type;
    PyObject **method_name;
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS
    __pyx_atomic_int_type initialized;
#endif
    PyCFunction func;
    PyObject *method;
    int flag;
} __Pyx_CachedCFunction;
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) {
#if !CYTHON_ATOMICS
    return 1;
#else
    __pyx_nonatomic_int_type expected = 0;
    if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) {
        return 0;
    }
    return expected;
#endif
}
static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) {
#if CYTHON_ATOMICS
    __pyx_atomic_store(&cfunc->initialized, 2);
#endif
}
#else
#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2
#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc)
#endif

/* CallUnboundCMethod0.proto */
CYTHON_UNUSED
static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self);
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self);
#else
#define __Pyx_CallUnboundCMethod0(cfunc, self)  __Pyx__CallUnboundCMethod0(cfunc, self)
#endif

/* py_dict_values.proto (used by OwnedDictNext) */
static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d);

/* OwnedDictNext.proto (used by RejectKeywords) */
#if CYTHON_AVOID_BORROWED_REFS
static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue);
#else
CYTHON_INLINE
static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue);
#endif

/* RejectKeywords.export */
static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds);

/* GetTopmostException.proto (used by SaveResetException) */
#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE
static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate);
#endif

/* SaveResetException.proto */
#if CYTHON_FAST_THREAD_STATE
#define __Pyx_ExceptionSave(type, value, tb)  __Pyx__ExceptionSave(__pyx_tstate, type, value, tb)
static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
#define __Pyx_ExceptionReset(type, value, tb)  __Pyx__ExceptionReset(__pyx_tstate, type, value, tb)
static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb);
#else
#define __Pyx_ExceptionSave(type, value, tb)   PyErr_GetExcInfo(type, value, tb)
#define __Pyx_ExceptionReset(type, value, tb)  PyErr_SetExcInfo(type, value, tb)
#endif

/* PyAttributeError_Check.proto */
#define __Pyx_PyExc_AttributeError_Check(obj)  __Pyx_TypeCheck(obj, PyExc_AttributeError)

/* GetException.proto */
#if CYTHON_FAST_THREAD_STATE
#define __Pyx_GetException(type, value, tb)  __Pyx__GetException(__pyx_tstate, type, value, tb)
static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
#else
static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb);
#endif

/* PyObjectDelAttr.proto (used by PyObjectSetAttrStr) */
#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000
#define __Pyx_PyObject_DelAttr(o, n) PyObject_SetAttr(o, n, NULL)
#else
#define __Pyx_PyObject_DelAttr(o, n) PyObject_DelAttr(o, n)
#endif

/* PyObjectSetAttrStr.proto */
#if CYTHON_USE_TYPE_SLOTS
#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o, n, NULL)
static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value);
#else
#define __Pyx_PyObject_DelAttrStr(o,n)   __Pyx_PyObject_DelAttr(o,n)
#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v)
#endif

/* GetAttr3.proto */
static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *);

/* PyDictVersioning.proto (used by GetModuleGlobalName) */
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS
#define __PYX_DICT_VERSION_INIT  ((PY_UINT64_T) -1)
#define __PYX_GET_DICT_VERSION(dict)  (((PyDictObject*)(dict))->ma_version_tag)
#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\
    (version_var) = __PYX_GET_DICT_VERSION(dict);\
    (cache_var) = (value);
#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\
    static PY_UINT64_T __pyx_dict_version = 0;\
    static PyObject *__pyx_dict_cached_value = NULL;\
    if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\
        (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\
    } else {\
        (VAR) = __pyx_dict_cached_value = (LOOKUP);\
        __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\
    }\
}
static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj);
static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj);
static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version);
#else
#define __PYX_GET_DICT_VERSION(dict)  (0)
#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)
#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP)  (VAR) = (LOOKUP);
#endif

/* GetModuleGlobalName.proto */
#if CYTHON_USE_DICT_VERSIONS
#define __Pyx_GetModuleGlobalName(var, name)  do {\
    static PY_UINT64_T __pyx_dict_version = 0;\
    static PyObject *__pyx_dict_cached_value = NULL;\
    (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\
        (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\
        __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\
} while(0)
#define __Pyx_GetModuleGlobalNameUncached(var, name)  do {\
    PY_UINT64_T __pyx_dict_version;\
    PyObject *__pyx_dict_cached_value;\
    (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\
} while(0)
static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value);
#else
#define __Pyx_GetModuleGlobalName(var, name)  (var) = __Pyx__GetModuleGlobalName(name)
#define __Pyx_GetModuleGlobalNameUncached(var, name)  (var) = __Pyx__GetModuleGlobalName(name)
static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name);
#endif

/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */
static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name);

/* ParseKeywordsImpl.export */
static int __Pyx_ParseKeywordsTuple(
    PyObject *kwds,
    PyObject * const *kwvalues,
    PyObject ** const argnames[],
    PyObject *kwds2,
    PyObject *values[],
    Py_ssize_t num_pos_args,
    Py_ssize_t num_kwargs,
    const char* function_name,
    int ignore_unknown_kwargs
);
static int __Pyx_ParseKeywordDictToDict(
    PyObject *kwds,
    PyObject ** const argnames[],
    PyObject *kwds2,
    PyObject *values[],
    Py_ssize_t num_pos_args,
    const char* function_name
);
static int __Pyx_ParseKeywordDict(
    PyObject *kwds,
    PyObject ** const argnames[],
    PyObject *values[],
    Py_ssize_t num_pos_args,
    Py_ssize_t num_kwargs,
    const char* function_name,
    int ignore_unknown_kwargs
);

/* CallUnboundCMethod2.proto */
CYTHON_UNUSED
static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2);
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2);
#else
#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2)  __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2)
#endif

/* ParseKeywords.proto */
static CYTHON_INLINE int __Pyx_ParseKeywords(
    PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[],
    PyObject *kwds2, PyObject *values[],
    Py_ssize_t num_pos_args, Py_ssize_t num_kwargs,
    const char* function_name,
    int ignore_unknown_kwargs
);

/* RaiseUnexpectedTypeError.proto */
static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj);

/* ImportImpl.export */
static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level);

/* Import.proto */
static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level);

/* ImportFrom.proto */
static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name);

/* PySequenceContains.proto */
static CYTHON_INLINE int __Pyx_PySequence_ContainsTF(PyObject* item, PyObject* seq, int eq) {
    int result = PySequence_Contains(seq, item);
    return unlikely(result < 0) ? result : (result == (eq == Py_EQ));
}

/* PyObject_Unicode.proto */
#define __Pyx_PyObject_Unicode(obj)\
    (likely(PyUnicode_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Str(obj))

/* SetItemInt.proto */
#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\
    (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
    __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\
    (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\
               __Pyx_SetItemInt_Generic(o, to_py_func(i), v)))
static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v);
static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v,
                                               int is_list, int wraparound, int boundscheck, int unsafe_shared);

/* PyRuntimeError_Check.proto */
#define __Pyx_PyExc_RuntimeError_Check(obj)  __Pyx_TypeCheck(obj, PyExc_RuntimeError)

/* PyValueError_Check.proto */
#define __Pyx_PyExc_ValueError_Check(obj)  __Pyx_TypeCheck(obj, PyExc_ValueError)

/* PyUnicodeContains.proto */
static CYTHON_INLINE int __Pyx_PyUnicode_ContainsTF(PyObject* substring, PyObject* text, int eq) {
    int result = PyUnicode_Contains(text, substring);
    return unlikely(result < 0) ? result : (result == (eq == Py_EQ));
}

/* PyRuntimeWarning_Check.proto */
#define __Pyx_PyExc_RuntimeWarning_Check(obj)  __Pyx_TypeCheck(obj, PyExc_RuntimeWarning)

/* PyObjectVectorCallKwBuilder.proto */
CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n);
#if CYTHON_VECTORCALL
#if PY_VERSION_HEX >= 0x03090000
#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall
#else
#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall
#endif
#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n)
static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n);
static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n);
#else
#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict
#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n)
#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value)
#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value)
#endif

/* PyUnicode_Unicode.proto */
static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj);

/* SetStringIndexingError.proto (used by GetItemIntUnicode) */
static void __Pyx_SetStringIndexingError(const char* message, int has_gil);

/* GetItemIntUnicode.proto */
#define __Pyx_GetItemInt_Unicode(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\
    (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
    __Pyx_GetItemInt_Unicode_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, has_gil) :\
    (__Pyx_SetStringIndexingError("string index out of range", has_gil), (Py_UCS4)-1))
static CYTHON_INLINE Py_UCS4 __Pyx_GetItemInt_Unicode_Fast(PyObject* ustring, Py_ssize_t i,
                                                           int wraparound, int boundscheck, int has_gil);

/* UnicodeConcatInPlace.proto */
# if CYTHON_COMPILING_IN_CPYTHON
    #if CYTHON_REFNANNY
        #define __Pyx_PyUnicode_ConcatInPlace(left, right, unsafe_shared) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, unsafe_shared, __pyx_refnanny)
    #else
        #define __Pyx_PyUnicode_ConcatInPlace(left, right, unsafe_shared) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, unsafe_shared)
    #endif
    #define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_DefinitelyUniqueInPlace(left, right) __Pyx_PyUnicode_ConcatInPlace(left, right, __Pyx_ReferenceSharing_DefinitelyUnique)
    #define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlace(left, right) __Pyx_PyUnicode_ConcatInPlace(left, right, __Pyx_ReferenceSharing_OwnStrongReference)
    #define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_FunctionArgumentInPlace(left, right) __Pyx_PyUnicode_ConcatInPlace(left, right, __Pyx_ReferenceSharing_DefinitelyUnique)
    #define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_SharedReferenceInPlace(left, right) __Pyx_PyUnicode_ConcatInPlace(left, right, __Pyx_ReferenceSharing_SharedReference)
    static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right, int unsafe_shared
        #if CYTHON_REFNANNY
        , void* __pyx_refnanny
        #endif
    );
#else
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_DefinitelyUniqueInPlace __Pyx_PyUnicode_Concat
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlace __Pyx_PyUnicode_Concat
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_FunctionArgumentInPlace __Pyx_PyUnicode_Concat
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_SharedReferenceInPlace __Pyx_PyUnicode_Concat
#endif
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_DefinitelyUniqueInPlaceSafe(left, right)\
    ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ?\
    PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_DefinitelyUniqueInPlace(left, right))
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlaceSafe(left, right)\
    ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ?\
    PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlace(left, right))
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_FunctionArgumentInPlaceSafe(left, right)\
    ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ?\
    PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_FunctionArgumentInPlace(left, right))
#define __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_SharedReferenceInPlaceSafe(left, right)\
    ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ?\
    PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_SharedReferenceInPlace(left, right))

/* PyUnicode_Substring.proto */
static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Substring(
            PyObject* text, Py_ssize_t start, Py_ssize_t stop);

/* DictGetItem.proto */
#if !CYTHON_COMPILING_IN_PYPY
static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key);
#define __Pyx_PyObject_Dict_GetItem(obj, name)\
    (likely(PyDict_CheckExact(obj)) ?\
     __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name))
#else
#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key)
#define __Pyx_PyObject_Dict_GetItem(obj, name)  PyObject_GetItem(obj, name)
#endif

/* ListAppend.proto */
#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS
static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) {
    PyListObject* L = (PyListObject*) list;
    Py_ssize_t len = Py_SIZE(list);
    if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) {
        Py_INCREF(x);
        #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000
        L->ob_item[len] = x;
        #else
        PyList_SET_ITEM(list, len, x);
        #endif
        __Pyx_SET_SIZE(list, len + 1);
        return 0;
    }
    return PyList_Append(list, x);
}
#else
#define __Pyx_PyList_Append(L,x) PyList_Append(L,x)
#endif

/* PyObjectLookupSpecial.proto */
#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name)  __Pyx__PyObject_LookupSpecial(obj, attr_name, 0)
#define __Pyx_PyObject_LookupSpecial(obj, attr_name)  __Pyx__PyObject_LookupSpecial(obj, attr_name, 1)
static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error);
#else
#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n)
#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n)
#endif

/* RaiseUnboundLocalError.proto */
static void __Pyx_RaiseUnboundLocalError(const char *varname);

/* PyObjectVectorCallMethodKwBuilder.proto */
#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000
#define __Pyx_Object_VectorcallMethod_CallFromBuilder PyObject_VectorcallMethod
#else
static PyObject *__Pyx_Object_VectorcallMethod_CallFromBuilder(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames);
#endif

/* FastTypeChecks.proto */
#if CYTHON_COMPILING_IN_CPYTHON
#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type)
#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2)
static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b);
static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b);
static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type);
static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2);
#else
#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type)
#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2))
#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type)
static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) {
    return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2);
}
#endif
#define __Pyx_PyErr_ExceptionMatches2(err1, err2)  __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2)
#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception)
#ifdef PyExceptionInstance_Check
  #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj)
#else
  #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException)
#endif

/* SliceTupleAndList.proto */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_PyList_GetSlice(PyObject* src, Py_ssize_t start, Py_ssize_t stop);
static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice(PyObject* src, Py_ssize_t start, Py_ssize_t stop);
#else
#define __Pyx_PyList_GetSlice(seq, start, stop)   PySequence_GetSlice(seq, start, stop)
#define __Pyx_PyTuple_GetSlice(seq, start, stop)  PySequence_GetSlice(seq, start, stop)
#endif

/* SwapException.proto */
#if CYTHON_FAST_THREAD_STATE
#define __Pyx_ExceptionSwap(type, value, tb)  __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb)
static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
#else
static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb);
#endif

/* ArgTypeTestFunc.export */
static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact);

/* ArgTypeTest.proto */
#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\
    ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\
        __Pyx__ArgTypeTest(obj, type, name, exact))

/* RaiseTooManyValuesToUnpack.proto */
static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected);

/* RaiseNeedMoreValuesToUnpack.proto */
static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index);

/* RaiseNoneIterError.proto */
static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void);

/* GetAttr.proto */
static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *, PyObject *);

/* PyNotImplementedError_Check.proto */
#define __Pyx_PyExc_NotImplementedError_Check(obj)  __Pyx_TypeCheck(obj, PyExc_NotImplementedError)

/* PyDeprecationWarning_Check.proto */
#define __Pyx_PyExc_DeprecationWarning_Check(obj)  __Pyx_TypeCheck(obj, PyExc_DeprecationWarning)

/* dict_getitem_default.proto */
static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value);

/* PyObjectCall2Args.proto (used by CallUnboundCMethod1) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2);

/* CallUnboundCMethod1.proto */
CYTHON_UNUSED
static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg);
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg);
#else
#define __Pyx_CallUnboundCMethod1(cfunc, self, arg)  __Pyx__CallUnboundCMethod1(cfunc, self, arg)
#endif

/* AssertionsEnabled.proto */
#if CYTHON_COMPILING_IN_LIMITED_API  ||  PY_VERSION_HEX >= 0x030C0000
  static int __pyx_assertions_enabled_flag;
  #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag)
  #if __clang__ || __GNUC__
  __attribute__((no_sanitize("thread")))
  #endif
  static int __Pyx_init_assertions_enabled(void) {
    PyObject *builtins, *debug, *debug_str;
    int flag;
    builtins = PyEval_GetBuiltins();
    if (!builtins) goto bad;
    debug_str = PyUnicode_FromStringAndSize("__debug__", 9);
    if (!debug_str) goto bad;
    debug = PyObject_GetItem(builtins, debug_str);
    Py_DECREF(debug_str);
    if (!debug) goto bad;
    flag = PyObject_IsTrue(debug);
    Py_DECREF(debug);
    if (flag == -1) goto bad;
    __pyx_assertions_enabled_flag = flag;
    return 0;
  bad:
    __pyx_assertions_enabled_flag = 1;
    return -1;
  }
#else
  #define __Pyx_init_assertions_enabled()  (0)
  #define __pyx_assertions_enabled()  (!Py_OptimizeFlag)
#endif

/* PyAssertionError_Check.proto */
#define __Pyx_PyExc_AssertionError_Check(obj)  __Pyx_TypeCheck(obj, PyExc_AssertionError)

/* AllocateExtensionType.proto */
static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final);

/* CallTypeTraverse.proto */
#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000)
#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0
#else
static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg);
#endif

/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */
#if CYTHON_COMPILING_IN_LIMITED_API
static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp);
#endif

/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */
static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v);
#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v)

/* FixUpExtensionType.proto */
static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type);

/* PyObjectCallNoArg.proto (used by PyObjectCallMethod0) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func);

/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */
#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)))
static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
#endif

/* PyObjectCallMethod0.proto (used by PyType_Ready) */
static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name);

/* ValidateBasesTuple.proto (used by PyType_Ready) */
#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS
static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases);
#endif

/* PyType_Ready.proto */
CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t);

/* SetVTable.proto */
static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable);

/* GetVTable.proto (used by MergeVTables) */
static void* __Pyx_GetVtable(PyTypeObject *type);

/* MergeVTables.proto */
static int __Pyx_MergeVtables(PyTypeObject *type);

/* DelItemOnTypeDict.proto (used by SetupReduce) */
static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k);
#define __Pyx_DelItemOnTypeDict(tp, k) __Pyx__DelItemOnTypeDict((PyTypeObject*)tp, k)

/* SetupReduce.proto */
static int __Pyx_setup_reduce(PyObject* type_obj);

/* TypeImport.proto */
#ifndef __PYX_HAVE_RT_ImportType_proto_3_2_5
#define __PYX_HAVE_RT_ImportType_proto_3_2_5
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
#include <stdalign.h>
#endif
#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L
#define __PYX_GET_STRUCT_ALIGNMENT_3_2_5(s) alignof(s)
#else
#define __PYX_GET_STRUCT_ALIGNMENT_3_2_5(s) sizeof(void*)
#endif
enum __Pyx_ImportType_CheckSize_3_2_5 {
   __Pyx_ImportType_CheckSize_Error_3_2_5 = 0,
   __Pyx_ImportType_CheckSize_Warn_3_2_5 = 1,
   __Pyx_ImportType_CheckSize_Ignore_3_2_5 = 2
};
static PyTypeObject *__Pyx_ImportType_3_2_5(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_5 check_size);
#endif

/* VoidPtrImport.proto */
static int __Pyx_ImportVoidPtr_3_2_5(PyObject *module, const char *name, void **p, const char *sig);

/* FunctionImport.proto */
static int __Pyx_ImportFunction_3_2_5(PyObject *module, const char *funcname, void (**f)(void), const char *sig);

/* dict_setdefault.proto (used by CLineInTraceback) */
static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value);

/* CLineInTraceback.proto (used by AddTraceback) */
#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME
static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line);
#else
#define __Pyx_CLineForTraceback(tstate, c_line)  (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0)
#endif

/* CodeObjectCache.proto (used by AddTraceback) */
#if CYTHON_COMPILING_IN_LIMITED_API
typedef PyObject __Pyx_CachedCodeObjectType;
#else
typedef PyCodeObject __Pyx_CachedCodeObjectType;
#endif
typedef struct {
    __Pyx_CachedCodeObjectType* code_object;
    int code_line;
} __Pyx_CodeObjectCacheEntry;
struct __Pyx_CodeObjectCache {
    int count;
    int max_count;
    __Pyx_CodeObjectCacheEntry* entries;
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    __pyx_atomic_int_type accessor_count;
  #endif
};
static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line);
static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line);
static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object);

/* AddTraceback.proto */
static void __Pyx_AddTraceback(const char *funcname, int c_line,
                               int py_line, const char *filename);

/* CheckUnpickleChecksum.proto */
static CYTHON_INLINE int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members);

/* GCCDiagnostics.proto */
#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
#define __Pyx_HAS_GCC_DIAGNOSTIC
#endif

/* CIntFromPy.proto */
static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *);

/* CIntToPy.proto */
static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value);

/* CIntFromPy.proto */
static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *);

/* CIntToPy.proto */
static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value);

/* PyObjectCallMethod1.proto */
static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg);

/* UpdateUnpickledDict.proto */
static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index);

/* FormatTypeName.proto */
#if CYTHON_COMPILING_IN_LIMITED_API
typedef PyObject *__Pyx_TypeName;
#define __Pyx_FMT_TYPENAME "%U"
#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj)
#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName
#else
static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp);
#endif
#else  // !LIMITED_API
typedef const char *__Pyx_TypeName;
#define __Pyx_FMT_TYPENAME "%.200s"
#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name)
#define __Pyx_DECREF_TypeName(obj)
#endif

/* GetRuntimeVersion.proto */
#if __PYX_LIMITED_VERSION_HEX < 0x030b0000
static unsigned long __Pyx_cached_runtime_version = 0;
static void __Pyx_init_runtime_version(void);
#else
#define __Pyx_init_runtime_version()
#endif
static unsigned long __Pyx_get_runtime_version(void);

/* AddModuleRef.proto */
#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\
     __PYX_LIMITED_VERSION_HEX < 0x030d0000)
  static PyObject *__Pyx_PyImport_AddModuleRef(const char *name);
#else
  #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name)
#endif

/* CheckBinaryVersion.proto */
static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer);

/* DecompressString.proto */
static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo);

/* MultiPhaseInitModuleState.proto */
#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE
static PyObject *__Pyx_State_FindModule(void*);
static int __Pyx_State_AddModule(PyObject* module, void*);
static int __Pyx_State_RemoveModule(void*);
#elif CYTHON_USE_MODULE_STATE
#define __Pyx_State_FindModule PyState_FindModule
#define __Pyx_State_AddModule PyState_AddModule
#define __Pyx_State_RemoveModule PyState_RemoveModule
#endif

/* #### Code section: module_declarations ### */
/* CythonABIVersion.proto */
#if CYTHON_COMPILING_IN_LIMITED_API
    #if CYTHON_METH_FASTCALL
        #define __PYX_FASTCALL_ABI_SUFFIX  "_fastcall"
    #else
        #define __PYX_FASTCALL_ABI_SUFFIX
    #endif
    #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX
#else
    #define __PYX_LIMITED_ABI_SUFFIX
#endif
#if __PYX_HAS_PY_AM_SEND == 1
    #define __PYX_AM_SEND_ABI_SUFFIX
#elif __PYX_HAS_PY_AM_SEND == 2
    #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport"
#else
    #define __PYX_AM_SEND_ABI_SUFFIX "noamsend"
#endif
#ifndef __PYX_MONITORING_ABI_SUFFIX
    #define __PYX_MONITORING_ABI_SUFFIX
#endif
#if CYTHON_USE_TP_FINALIZE
    #define __PYX_TP_FINALIZE_ABI_SUFFIX
#else
    #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize"
#endif
#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED)
    #define __PYX_FREELISTS_ABI_SUFFIX
#else
    #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists"
#endif
#define CYTHON_ABI  __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX
#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI
#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "."

static CYTHON_INLINE PyObject *__pyx_f_4cupy_4cuda_6memory_10MemoryPool_device_pool(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *__pyx_v_self); /* proto*/
static struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_f_4cupy_3fft_9_callback_12_ThreadLocal_get(void); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self, struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos *__pyx_optional_args); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_create_plan(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan_info, int __pyx_skip_dispatch); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan, int __pyx_skip_dispatch); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager__get_cuda_include(CYTHON_UNUSED struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager__sanity_checks(CYTHON_UNUSED struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, PyObject *__pyx_v_cb_load_data, PyObject *__pyx_v_cb_store_data); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_compile_lto(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_source, PyObject *__pyx_v_options); /* proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_create_plan(CYTHON_UNUSED struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, intptr_t __pyx_v_prealloc_plan, PyObject *__pyx_v_plan_info, int __pyx_skip_dispatch); /* proto*/
static intptr_t __pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_fft_type, int __pyx_skip_dispatch); /* proto*/

/* Module declarations from "libc.stdint" */

/* Module declarations from "cupy_backends.cuda.api" */

/* Module declarations from "libcpp.vector" */

/* Module declarations from "cupy_backends.cuda.api._driver_enum" */

/* Module declarations from "cupy_backends.cuda.api.driver" */

/* Module declarations from "cupy_backends.cuda.api._runtime_enum" */

/* Module declarations from "cupy_backends.cuda.api.runtime" */
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorMemoryAllocation = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_errorMemoryAllocation (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorMemoryAllocation)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorInvalidValue = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_errorInvalidValue (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorInvalidValue)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorPeerAccessAlreadyEnabled = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_errorPeerAccessAlreadyEnabled (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorPeerAccessAlreadyEnabled)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorContextIsDestroyed = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_errorContextIsDestroyed (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorContextIsDestroyed)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorInvalidResourceHandle = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_errorInvalidResourceHandle (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorInvalidResourceHandle)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMajor = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMajor (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMajor)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMinor = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMinor (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMinor)
static int *__pyx_vp_13cupy_backends_4cuda_3api_7runtime__is_hip_environment = 0;
#define __pyx_v_13cupy_backends_4cuda_3api_7runtime__is_hip_environment (*__pyx_vp_13cupy_backends_4cuda_3api_7runtime__is_hip_environment)
static int (*__pyx_f_13cupy_backends_4cuda_3api_7runtime_runtimeGetVersion)(int __pyx_skip_dispatch); /*proto*/

/* Module declarations from "libcpp" */

/* Module declarations from "cupy.cuda" */

/* Module declarations from "cython" */

/* Module declarations from "libcpp.utility" */

/* Module declarations from "libcpp.map" */

/* Module declarations from "cupy.cuda.device" */
static int (*__pyx_f_4cupy_4cuda_6device_get_device_id)(int __pyx_skip_dispatch); /*proto*/
static PyObject *(*__pyx_f_4cupy_4cuda_6device_get_compute_capability)(int __pyx_skip_dispatch); /*proto*/

/* Module declarations from "cupy.cuda.memory" */

/* Module declarations from "cupy.cuda.function" */

/* Module declarations from "cupy._core._carray" */

/* Module declarations from "cupy._core.core" */
static struct __pyx_obj_4cupy_5_core_4core__ndarray_base *(*__pyx_f_4cupy_5_core_4core__convert_object_with_cuda_array_interface)(PyObject *, int __pyx_skip_dispatch); /*proto*/

/* Module declarations from "cupy.fft._callback" */
static PyObject *__pyx_v_4cupy_3fft_9_callback__cc = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__python_include = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__nvcc = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__cuda_path = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__cuda_include = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__nvprune = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__build_ver = 0;
static int __pyx_v_4cupy_3fft_9_callback__cufft_ver;
static PyObject *__pyx_v_4cupy_3fft_9_callback__cupy_root = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__cupy_include = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__source_dir = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__ext_suffix = 0;
static int __pyx_v_4cupy_3fft_9_callback__is_init;
static PyObject *__pyx_v_4cupy_3fft_9_callback__callback_dev_code = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__callback_cache_dir = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__callback_mgr = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__callback_thread_local = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback__cc_major_map = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback_cupy_load_callback_pattern = 0;
static PyObject *__pyx_v_4cupy_3fft_9_callback_cupy_store_callback_pattern = 0;
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__set_vars(void); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__set_cupy_paths(void); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__set_nvcc_path(void); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__sanity_checks(PyObject *, PyObject *, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__cythonize(PyObject *, PyObject *); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__mod_compile(PyObject *, PyObject *, PyObject *); /*proto*/
static CYTHON_INLINE PyObject *__pyx_f_4cupy_3fft_9_callback__prune(PyObject *, PyObject *, PyObject *, PyObject *); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__nvcc_compile(PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); /*proto*/
static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__nvcc_link(PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_get_current_callback_manager(int __pyx_skip_dispatch); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__ThreadLocal__set_state(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *, PyObject *); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__CallbackManager__set_state(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *, PyObject *); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__LegacyCallbackManager__set_state(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *, PyObject *); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__JITCallbackManager__set_state(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle_set_cufft_callbacks__set_state(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *, PyObject *); /*proto*/
/* #### Code section: typeinfo ### */
/* #### Code section: before_global_var ### */
#define __Pyx_MODULE_NAME "cupy.fft._callback"
extern int __pyx_module_is_main_cupy__fft___callback;
int __pyx_module_is_main_cupy__fft___callback = 0;

/* Implementation of "cupy.fft._callback" */
/* #### Code section: global_var ### */
static PyObject *__pyx_builtin_enumerate;
static PyObject *__pyx_builtin_open;
/* #### Code section: string_decls ### */
static const char __pyx_k_mgr_mgr_prev[] = "mgr, mgr_prev";
static const char __pyx_k_current_cufft_callback[] = "_current_cufft_callback";
static const char __pyx_k_cb_load_cb_load_data_cb_load_lto[] = "cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity";
static const char __pyx_k_cb_load_cb_load_data_cb_store_cb[] = "cb_load, cb_load_data, cb_store, cb_store_data, identity, mod";
static const char __pyx_k_cb_load_data_cb_store_data_ident[] = "cb_load_data, cb_store_data, identity";
/* #### Code section: decls ### */
static int __pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal___init__(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal_2__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal_4__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_get_cache_dir(CYTHON_UNUSED PyObject *__pyx_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_2get_current_callback_manager(CYTHON_UNUSED PyObject *__pyx_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_12cb_load_data___get__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_13cb_store_data___get__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_8identity___get__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager___reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_2__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */
static int __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager___init__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_2create_plan(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan_info); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_4set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_7cb_load___get__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_8cb_store___get__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_6__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_8__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */
static int __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager___init__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, PyObject *__pyx_v_cb_load_name, PyObject *__pyx_v_cb_store_name, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data, PyObject *__pyx_v_nvrtc_options); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_2create_plan(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, intptr_t __pyx_v_prealloc_plan, PyObject *__pyx_v_plan_info); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_4set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_fft_type); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_7cb_load___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_8cb_store___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_load_name___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_13cb_store_name___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_11cb_load_lto___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_store_lto___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_6__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_8__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */
static int __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks___init__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_v_cb_load_aux_arr, struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_v_cb_store_aux_arr, PyObject *__pyx_v_cb_load_name, PyObject *__pyx_v_cb_store_name, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data, PyObject *__pyx_v_cb_ver, PyObject *__pyx_v_nvrtc_options); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_2__enter__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_4__exit__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_exc_type, CYTHON_UNUSED PyObject *__pyx_v_exc_value, CYTHON_UNUSED PyObject *__pyx_v_traceback); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_6__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_8__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_4__pyx_unpickle__ThreadLocal(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_6__pyx_unpickle__CallbackManager(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_8__pyx_unpickle__LegacyCallbackManager(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_10__pyx_unpickle__JITCallbackManager(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_pf_4cupy_3fft_9_callback_12__pyx_unpickle_set_cufft_callbacks(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */
static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__ThreadLocal(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__CallbackManager(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__LegacyCallbackManager(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__JITCallbackManager(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
static PyObject *__pyx_tp_new_4cupy_3fft_9_callback_set_cufft_callbacks(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
/* #### Code section: late_includes ### */
/* #### Code section: module_state ### */
/* SmallCodeConfig */
#ifndef CYTHON_SMALL_CODE
#if defined(__clang__)
    #define CYTHON_SMALL_CODE
#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
    #define CYTHON_SMALL_CODE __attribute__((cold))
#else
    #define CYTHON_SMALL_CODE
#endif
#endif

typedef struct {
  PyObject *__pyx_d;
  PyObject *__pyx_b;
  PyObject *__pyx_cython_runtime;
  PyObject *__pyx_empty_tuple;
  PyObject *__pyx_empty_bytes;
  PyObject *__pyx_empty_unicode;
  PyTypeObject *__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_PointerAttributes;
  PyTypeObject *__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_MemPoolProps;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6device_Handle;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6device_Device;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_BaseMemory;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_SystemMemory;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_MemoryPool;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocatorMemory;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocator;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocator;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_8function_CPointer;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_8function_Function;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_8function_Module;
  PyTypeObject *__pyx_ptype_4cupy_4cuda_8function_LinkState;
  PyTypeObject *__pyx_ptype_4cupy_5_core_7_carray_mdspan;
  PyTypeObject *__pyx_ptype_4cupy_5_core_7_carray_CArray;
  PyTypeObject *__pyx_ptype_4cupy_5_core_7_carray_CIndexer;
  PyTypeObject *__pyx_ptype_4cupy_5_core_7_carray_Indexer;
  PyTypeObject *__pyx_ptype_4cupy_5_core_4core__ndarray_base;
  PyObject *__pyx_type_4cupy_3fft_9_callback__ThreadLocal;
  PyObject *__pyx_type_4cupy_3fft_9_callback__CallbackManager;
  PyObject *__pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager;
  PyObject *__pyx_type_4cupy_3fft_9_callback__JITCallbackManager;
  PyObject *__pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks;
  PyTypeObject *__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal;
  PyTypeObject *__pyx_ptype_4cupy_3fft_9_callback__CallbackManager;
  PyTypeObject *__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager;
  PyTypeObject *__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager;
  PyTypeObject *__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks;
  __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_get;
  __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items;
  __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop;
  __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values;
  PyObject *__pyx_tuple[9];
  PyObject *__pyx_string_tab[301];
  PyObject *__pyx_number_tab[10];
/* #### Code section: module_state_contents ### */
/* CodeObjectCache.module_state_decls */
struct __Pyx_CodeObjectCache __pyx_code_cache;

/* #### Code section: module_state_end ### */
} __pyx_mstatetype;

#if CYTHON_USE_MODULE_STATE
#ifdef __cplusplus
namespace {
extern struct PyModuleDef __pyx_moduledef;
} /* anonymous namespace */
#else
static struct PyModuleDef __pyx_moduledef;
#endif

#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef)))

#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef))
#else
static __pyx_mstatetype __pyx_mstate_global_static =
#ifdef __cplusplus
    {};
#else
    {0};
#endif
static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static;
#endif
/* #### Code section: constant_name_defines ### */
#define __pyx_kp_u_ __pyx_string_tab[0]
#define __pyx_kp_u_10 __pyx_string_tab[1]
#define __pyx_kp_u_100 __pyx_string_tab[2]
#define __pyx_kp_u_103 __pyx_string_tab[3]
#define __pyx_kp_u_12 __pyx_string_tab[4]
#define __pyx_kp_u_120 __pyx_string_tab[5]
#define __pyx_kp_u_121 __pyx_string_tab[6]
#define __pyx_kp_u_3 __pyx_string_tab[7]
#define __pyx_kp_u_5 __pyx_string_tab[8]
#define __pyx_kp_u_50 __pyx_string_tab[9]
#define __pyx_kp_u_52 __pyx_string_tab[10]
#define __pyx_kp_u_53 __pyx_string_tab[11]
#define __pyx_kp_u_6 __pyx_string_tab[12]
#define __pyx_kp_u_60 __pyx_string_tab[13]
#define __pyx_kp_u_61 __pyx_string_tab[14]
#define __pyx_kp_u_62 __pyx_string_tab[15]
#define __pyx_kp_u_7 __pyx_string_tab[16]
#define __pyx_kp_u_70 __pyx_string_tab[17]
#define __pyx_kp_u_72 __pyx_string_tab[18]
#define __pyx_kp_u_75 __pyx_string_tab[19]
#define __pyx_kp_u_8 __pyx_string_tab[20]
#define __pyx_kp_u_80 __pyx_string_tab[21]
#define __pyx_kp_u_86 __pyx_string_tab[22]
#define __pyx_kp_u_87 __pyx_string_tab[23]
#define __pyx_kp_u_9 __pyx_string_tab[24]
#define __pyx_kp_u_90 __pyx_string_tab[25]
#define __pyx_kp_u_Bsymbolic __pyx_string_tab[26]
#define __pyx_kp_u_CUPY_CUDA_VERSION __pyx_string_tab[27]
#define __pyx_kp_u_CUPY_CUFFT_STATIC_True __pyx_string_tab[28]
#define __pyx_kp_u_CUPY_HIP_VERSION_0 __pyx_string_tab[29]
#define __pyx_kp_u_DHAS_LOAD_CALLBACK __pyx_string_tab[30]
#define __pyx_kp_u_DHAS_STORE_CALLBACK __pyx_string_tab[31]
#define __pyx_kp_u_E __pyx_string_tab[32]
#define __pyx_kp_u_Failed_to_find_CUDA_headers __pyx_string_tab[33]
#define __pyx_kp_u_I __pyx_string_tab[34]
#define __pyx_kp_u_JIT_callback_support_requires_cu __pyx_string_tab[35]
#define __pyx_kp_u_L __pyx_string_tab[36]
#define __pyx_kp_u_None __pyx_string_tab[37]
#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[38]
#define __pyx_kp_u_O2 __pyx_string_tab[39]
#define __pyx_kp_u_Stderr __pyx_string_tab[40]
#define __pyx_kp_u_Using_the_legacy_callback_is_con __pyx_string_tab[41]
#define __pyx_kp_u_Xcompiler __pyx_string_tab[42]
#define __pyx_kp_u_Xlinker __pyx_string_tab[43]
#define __pyx_kp_u__2 __pyx_string_tab[44]
#define __pyx_kp_u__3 __pyx_string_tab[45]
#define __pyx_kp_u__4 __pyx_string_tab[46]
#define __pyx_kp_u__5 __pyx_string_tab[47]
#define __pyx_kp_u__6 __pyx_string_tab[48]
#define __pyx_kp_u__7 __pyx_string_tab[49]
#define __pyx_kp_u_a __pyx_string_tab[50]
#define __pyx_kp_u_a_C_compiler_is_required_but_not __pyx_string_tab[51]
#define __pyx_kp_u_add_note __pyx_string_tab[52]
#define __pyx_kp_u_arch_sm __pyx_string_tab[53]
#define __pyx_kp_u_bin_nvprune __pyx_string_tab[54]
#define __pyx_kp_u_c __pyx_string_tab[55]
#define __pyx_kp_u_cb_load_name_cannot_be_inferred __pyx_string_tab[56]
#define __pyx_kp_u_cb_store_name_cannot_be_inferred __pyx_string_tab[57]
#define __pyx_kp_u_cb_ver_must_be_legacy_or_jit __pyx_string_tab[58]
#define __pyx_kp_u_ccbin __pyx_string_tab[59]
#define __pyx_kp_u_code_sm __pyx_string_tab[60]
#define __pyx_kp_u_compiler_bindir __pyx_string_tab[61]
#define __pyx_kp_u_core_include __pyx_string_tab[62]
#define __pyx_kp_u_cplus __pyx_string_tab[63]
#define __pyx_kp_u_cpp __pyx_string_tab[64]
#define __pyx_kp_u_cuFFT_callbacks_are_only_availab __pyx_string_tab[65]
#define __pyx_kp_u_cuFFT_callbacks_require_64_bit_O __pyx_string_tab[66]
#define __pyx_kp_u_cufft_pxd __pyx_string_tab[67]
#define __pyx_kp_u_cufft_pyx __pyx_string_tab[68]
#define __pyx_kp_u_cupy_callback_cache __pyx_string_tab[69]
#define __pyx_kp_u_cupy_cufftXt_cu __pyx_string_tab[70]
#define __pyx_kp_u_cupy_cufftXt_h __pyx_string_tab[71]
#define __pyx_kp_u_cupy_cufft_h __pyx_string_tab[72]
#define __pyx_kp_u_d_loadCallbackPtr_s_s_A_Za_z__w __pyx_string_tab[73]
#define __pyx_kp_u_d_storeCallbackPtr_s_s_A_Za_z__w __pyx_string_tab[74]
#define __pyx_kp_u_dc __pyx_string_tab[75]
#define __pyx_kp_u_dev_o __pyx_string_tab[76]
#define __pyx_kp_u_disable __pyx_string_tab[77]
#define __pyx_kp_u_enable __pyx_string_tab[78]
#define __pyx_kp_u_fPIC __pyx_string_tab[79]
#define __pyx_kp_u_gc __pyx_string_tab[80]
#define __pyx_kp_u_generate_code_arch_compute __pyx_string_tab[81]
#define __pyx_kp_u_got __pyx_string_tab[82]
#define __pyx_kp_u_has_incorrect_type_expected __pyx_string_tab[83]
#define __pyx_kp_u_hipFFT_does_not_support_callback __pyx_string_tab[84]
#define __pyx_kp_u_isenabled __pyx_string_tab[85]
#define __pyx_kp_u_l __pyx_string_tab[86]
#define __pyx_kp_u_lcufft_static __pyx_string_tab[87]
#define __pyx_kp_u_lculibos __pyx_string_tab[88]
#define __pyx_kp_u_lib64_libcufft_static_a __pyx_string_tab[89]
#define __pyx_kp_u_load_callback_is_not_given __pyx_string_tab[90]
#define __pyx_kp_u_lpthread __pyx_string_tab[91]
#define __pyx_kp_u_need_to_specify_d_loadCallbackPt __pyx_string_tab[92]
#define __pyx_kp_u_need_to_specify_d_storeCallbackP __pyx_string_tab[93]
#define __pyx_kp_u_need_to_specify_either_cb_load_o __pyx_string_tab[94]
#define __pyx_kp_u_nvcc_is_required_but_not_found __pyx_string_tab[95]
#define __pyx_kp_u_nvprune_failed_with_sm __pyx_string_tab[96]
#define __pyx_kp_u_nvprune_is_not_found __pyx_string_tab[97]
#define __pyx_kp_u_o __pyx_string_tab[98]
#define __pyx_kp_u_o_2 __pyx_string_tab[99]
#define __pyx_kp_u_only_use_cb_load_data_and_not_cb __pyx_string_tab[100]
#define __pyx_kp_u_only_use_cb_store_data_and_not_c __pyx_string_tab[101]
#define __pyx_kp_u_passing_an_ndarray_to_cb_load_au __pyx_string_tab[102]
#define __pyx_kp_u_pxd __pyx_string_tab[103]
#define __pyx_kp_u_pyx __pyx_string_tab[104]
#define __pyx_kp_u_s_s_s_s_s_s_s_s_s __pyx_string_tab[105]
#define __pyx_kp_u_shared __pyx_string_tab[106]
#define __pyx_kp_u_std_c_17 __pyx_string_tab[107]
#define __pyx_kp_u_std_c_17_2 __pyx_string_tab[108]
#define __pyx_kp_u_store_callback_is_not_given __pyx_string_tab[109]
#define __pyx_kp_u_when_both_cb_load_and_cb_store_a __pyx_string_tab[110]
#define __pyx_n_u_CUFFT_C2C __pyx_string_tab[111]
#define __pyx_n_u_CUFFT_C2R __pyx_string_tab[112]
#define __pyx_n_u_CUFFT_CB_LD_COMPLEX __pyx_string_tab[113]
#define __pyx_n_u_CUFFT_CB_LD_COMPLEX_DOUBLE __pyx_string_tab[114]
#define __pyx_n_u_CUFFT_CB_LD_REAL __pyx_string_tab[115]
#define __pyx_n_u_CUFFT_CB_LD_REAL_DOUBLE __pyx_string_tab[116]
#define __pyx_n_u_CUFFT_CB_ST_COMPLEX __pyx_string_tab[117]
#define __pyx_n_u_CUFFT_CB_ST_COMPLEX_DOUBLE __pyx_string_tab[118]
#define __pyx_n_u_CUFFT_CB_ST_REAL __pyx_string_tab[119]
#define __pyx_n_u_CUFFT_CB_ST_REAL_DOUBLE __pyx_string_tab[120]
#define __pyx_n_u_CUFFT_D2Z __pyx_string_tab[121]
#define __pyx_n_u_CUFFT_R2C __pyx_string_tab[122]
#define __pyx_n_u_CUFFT_Z2D __pyx_string_tab[123]
#define __pyx_n_u_CUFFT_Z2Z __pyx_string_tab[124]
#define __pyx_n_u_CUPY_CACHE_DIR __pyx_string_tab[125]
#define __pyx_n_u_CUPY_DUMP_CUDA_SOURCE_ON_ERROR __pyx_string_tab[126]
#define __pyx_n_u_CXX __pyx_string_tab[127]
#define __pyx_n_u_CallbackManager __pyx_string_tab[128]
#define __pyx_n_u_CalledProcessError __pyx_string_tab[129]
#define __pyx_n_u_CompileException __pyx_string_tab[130]
#define __pyx_n_u_EXT_SUFFIX __pyx_string_tab[131]
#define __pyx_n_u_JITCallbackManager __pyx_string_tab[132]
#define __pyx_n_u_LegacyCallbackManager __pyx_string_tab[133]
#define __pyx_n_u_PIPE __pyx_string_tab[134]
#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[135]
#define __pyx_n_u_Template __pyx_string_tab[136]
#define __pyx_n_u_TemporaryDirectory __pyx_string_tab[137]
#define __pyx_n_u_ThreadLocal __pyx_string_tab[138]
#define __pyx_n_u_cb_load __pyx_string_tab[139]
#define __pyx_n_u_cb_load_aux_arr __pyx_string_tab[140]
#define __pyx_n_u_cb_load_data __pyx_string_tab[141]
#define __pyx_n_u_cb_load_name __pyx_string_tab[142]
#define __pyx_n_u_cb_store __pyx_string_tab[143]
#define __pyx_n_u_cb_store_aux_arr __pyx_string_tab[144]
#define __pyx_n_u_cb_store_data __pyx_string_tab[145]
#define __pyx_n_u_cb_store_name __pyx_string_tab[146]
#define __pyx_n_u_cb_ver __pyx_string_tab[147]
#define __pyx_n_u_check_returncode __pyx_string_tab[148]
#define __pyx_n_u_cleanup __pyx_string_tab[149]
#define __pyx_n_u_cline_in_traceback __pyx_string_tab[150]
#define __pyx_n_u_compile __pyx_string_tab[151]
#define __pyx_n_u_compile_with_cache_cuda __pyx_string_tab[152]
#define __pyx_n_u_copyfile __pyx_string_tab[153]
#define __pyx_n_u_create __pyx_string_tab[154]
#define __pyx_n_u_create_plan __pyx_string_tab[155]
#define __pyx_n_u_cuda __pyx_string_tab[156]
#define __pyx_n_u_cuda_array_interface __pyx_string_tab[157]
#define __pyx_n_u_cuda_pathfinder __pyx_string_tab[158]
#define __pyx_n_u_cudart __pyx_string_tab[159]
#define __pyx_n_u_cufft __pyx_string_tab[160]
#define __pyx_n_u_cufft_static __pyx_string_tab[161]
#define __pyx_n_u_cupy __pyx_string_tab[162]
#define __pyx_n_u_cupy__environment __pyx_string_tab[163]
#define __pyx_n_u_cupy_callback __pyx_string_tab[164]
#define __pyx_n_u_cupy_cuda __pyx_string_tab[165]
#define __pyx_n_u_cupy_cuda_compiler __pyx_string_tab[166]
#define __pyx_n_u_cupy_cuda_cufft __pyx_string_tab[167]
#define __pyx_n_u_cupy_fft__callback __pyx_string_tab[168]
#define __pyx_n_u_cupy_get_ndarray __pyx_string_tab[169]
#define __pyx_n_u_cupy_ver __pyx_string_tab[170]
#define __pyx_n_u_cwd __pyx_string_tab[171]
#define __pyx_n_u_cython __pyx_string_tab[172]
#define __pyx_n_u_d_loadCallbackPtr __pyx_string_tab[173]
#define __pyx_n_u_d_storeCallbackPtr __pyx_string_tab[174]
#define __pyx_n_u_decode __pyx_string_tab[175]
#define __pyx_n_u_dev_load_callback_ker __pyx_string_tab[176]
#define __pyx_n_u_dev_store_callback_ker __pyx_string_tab[177]
#define __pyx_n_u_dict __pyx_string_tab[178]
#define __pyx_n_u_dir __pyx_string_tab[179]
#define __pyx_n_u_dirname __pyx_string_tab[180]
#define __pyx_n_u_dump __pyx_string_tab[181]
#define __pyx_n_u_encode __pyx_string_tab[182]
#define __pyx_n_u_enter __pyx_string_tab[183]
#define __pyx_n_u_enumerate __pyx_string_tab[184]
#define __pyx_n_u_env __pyx_string_tab[185]
#define __pyx_n_u_environ __pyx_string_tab[186]
#define __pyx_n_u_exc_type __pyx_string_tab[187]
#define __pyx_n_u_exc_value __pyx_string_tab[188]
#define __pyx_n_u_exec_module __pyx_string_tab[189]
#define __pyx_n_u_exist_ok __pyx_string_tab[190]
#define __pyx_n_u_exit __pyx_string_tab[191]
#define __pyx_n_u_expanduser __pyx_string_tab[192]
#define __pyx_n_u_fft_type __pyx_string_tab[193]
#define __pyx_n_u_file __pyx_string_tab[194]
#define __pyx_n_u_find_nvidia_binary_utility __pyx_string_tab[195]
#define __pyx_n_u_find_nvidia_header_directory __pyx_string_tab[196]
#define __pyx_n_u_get __pyx_string_tab[197]
#define __pyx_n_u_getVersion __pyx_string_tab[198]
#define __pyx_n_u_get_bool_env_variable __pyx_string_tab[199]
#define __pyx_n_u_get_cache_dir __pyx_string_tab[200]
#define __pyx_n_u_get_config_var __pyx_string_tab[201]
#define __pyx_n_u_get_cuda_path __pyx_string_tab[202]
#define __pyx_n_u_get_nvcc_path __pyx_string_tab[203]
#define __pyx_n_u_get_path __pyx_string_tab[204]
#define __pyx_n_u_getstate __pyx_string_tab[205]
#define __pyx_n_u_group __pyx_string_tab[206]
#define __pyx_n_u_handle __pyx_string_tab[207]
#define __pyx_n_u_hashlib __pyx_string_tab[208]
#define __pyx_n_u_hexdigest __pyx_string_tab[209]
#define __pyx_n_u_importlib __pyx_string_tab[210]
#define __pyx_n_u_include __pyx_string_tab[211]
#define __pyx_n_u_isdir __pyx_string_tab[212]
#define __pyx_n_u_isfile __pyx_string_tab[213]
#define __pyx_n_u_items __pyx_string_tab[214]
#define __pyx_n_u_jit __pyx_string_tab[215]
#define __pyx_n_u_join __pyx_string_tab[216]
#define __pyx_n_u_legacy __pyx_string_tab[217]
#define __pyx_n_u_lib __pyx_string_tab[218]
#define __pyx_n_u_linux __pyx_string_tab[219]
#define __pyx_n_u_loader __pyx_string_tab[220]
#define __pyx_n_u_local __pyx_string_tab[221]
#define __pyx_n_u_main __pyx_string_tab[222]
#define __pyx_n_u_makedirs __pyx_string_tab[223]
#define __pyx_n_u_maxsize __pyx_string_tab[224]
#define __pyx_n_u_md5 __pyx_string_tab[225]
#define __pyx_n_u_module __pyx_string_tab[226]
#define __pyx_n_u_module_from_spec __pyx_string_tab[227]
#define __pyx_n_u_modules __pyx_string_tab[228]
#define __pyx_n_u_name __pyx_string_tab[229]
#define __pyx_n_u_name_2 __pyx_string_tab[230]
#define __pyx_n_u_new __pyx_string_tab[231]
#define __pyx_n_u_nvcc __pyx_string_tab[232]
#define __pyx_n_u_nvprune __pyx_string_tab[233]
#define __pyx_n_u_nvrtc_options __pyx_string_tab[234]
#define __pyx_n_u_open __pyx_string_tab[235]
#define __pyx_n_u_os __pyx_string_tab[236]
#define __pyx_n_u_path __pyx_string_tab[237]
#define __pyx_n_u_plan __pyx_string_tab[238]
#define __pyx_n_u_plan_info __pyx_string_tab[239]
#define __pyx_n_u_platform __pyx_string_tab[240]
#define __pyx_n_u_pop __pyx_string_tab[241]
#define __pyx_n_u_prealloc_plan __pyx_string_tab[242]
#define __pyx_n_u_pyx_checksum __pyx_string_tab[243]
#define __pyx_n_u_pyx_state __pyx_string_tab[244]
#define __pyx_n_u_pyx_type __pyx_string_tab[245]
#define __pyx_n_u_pyx_unpickle__CallbackManager __pyx_string_tab[246]
#define __pyx_n_u_pyx_unpickle__JITCallbackManag __pyx_string_tab[247]
#define __pyx_n_u_pyx_unpickle__LegacyCallbackMa __pyx_string_tab[248]
#define __pyx_n_u_pyx_unpickle__ThreadLocal __pyx_string_tab[249]
#define __pyx_n_u_pyx_unpickle_set_cufft_callbac __pyx_string_tab[250]
#define __pyx_n_u_pyx_vtable __pyx_string_tab[251]
#define __pyx_n_u_qualname __pyx_string_tab[252]
#define __pyx_n_u_re __pyx_string_tab[253]
#define __pyx_n_u_read __pyx_string_tab[254]
#define __pyx_n_u_reduce __pyx_string_tab[255]
#define __pyx_n_u_reduce_cython __pyx_string_tab[256]
#define __pyx_n_u_reduce_ex __pyx_string_tab[257]
#define __pyx_n_u_rename __pyx_string_tab[258]
#define __pyx_n_u_run __pyx_string_tab[259]
#define __pyx_n_u_search __pyx_string_tab[260]
#define __pyx_n_u_setAutoAllocation __pyx_string_tab[261]
#define __pyx_n_u_setCallback __pyx_string_tab[262]
#define __pyx_n_u_setJITCallback __pyx_string_tab[263]
#define __pyx_n_u_set_callbacks __pyx_string_tab[264]
#define __pyx_n_u_set_cufft_callbacks __pyx_string_tab[265]
#define __pyx_n_u_set_name __pyx_string_tab[266]
#define __pyx_n_u_setdefault __pyx_string_tab[267]
#define __pyx_n_u_setstate __pyx_string_tab[268]
#define __pyx_n_u_setstate_cython __pyx_string_tab[269]
#define __pyx_n_u_shutil __pyx_string_tab[270]
#define __pyx_n_u_sm __pyx_string_tab[271]
#define __pyx_n_u_spec_from_file_location __pyx_string_tab[272]
#define __pyx_n_u_split __pyx_string_tab[273]
#define __pyx_n_u_startswith __pyx_string_tab[274]
#define __pyx_n_u_stderr __pyx_string_tab[275]
#define __pyx_n_u_stdout __pyx_string_tab[276]
#define __pyx_n_u_string __pyx_string_tab[277]
#define __pyx_n_u_subprocess __pyx_string_tab[278]
#define __pyx_n_u_substitute __pyx_string_tab[279]
#define __pyx_n_u_sys __pyx_string_tab[280]
#define __pyx_n_u_sysconfig __pyx_string_tab[281]
#define __pyx_n_u_tempfile __pyx_string_tab[282]
#define __pyx_n_u_test __pyx_string_tab[283]
#define __pyx_n_u_threading __pyx_string_tab[284]
#define __pyx_n_u_tls __pyx_string_tab[285]
#define __pyx_n_u_to_ltoir __pyx_string_tab[286]
#define __pyx_n_u_traceback __pyx_string_tab[287]
#define __pyx_n_u_update __pyx_string_tab[288]
#define __pyx_n_u_usedforsecurity __pyx_string_tab[289]
#define __pyx_n_u_util __pyx_string_tab[290]
#define __pyx_n_u_values __pyx_string_tab[291]
#define __pyx_n_u_version __pyx_string_tab[292]
#define __pyx_n_u_w __pyx_string_tab[293]
#define __pyx_n_u_warn __pyx_string_tab[294]
#define __pyx_n_u_warnings __pyx_string_tab[295]
#define __pyx_n_u_write __pyx_string_tab[296]
#define __pyx_kp_b_PyObject_int___pyx_skip_dispatch __pyx_string_tab[297]
#define __pyx_kp_b_int__is_hip_environment_deviceAt __pyx_string_tab[298]
#define __pyx_kp_b_int_int___pyx_skip_dispatch_runt __pyx_string_tab[299]
#define __pyx_kp_b_struct___pyx_obj_4cupy_5_core_4c __pyx_string_tab[300]
#define __pyx_int_0 __pyx_number_tab[0]
#define __pyx_int_neg_1 __pyx_number_tab[1]
#define __pyx_int_1 __pyx_number_tab[2]
#define __pyx_int_11303 __pyx_number_tab[3]
#define __pyx_int_52696597 __pyx_number_tab[4]
#define __pyx_int_146004246 __pyx_number_tab[5]
#define __pyx_int_238673578 __pyx_number_tab[6]
#define __pyx_int_258762224 __pyx_number_tab[7]
#define __pyx_int_265177976 __pyx_number_tab[8]
#define __pyx_int_4294967296 __pyx_number_tab[9]
/* #### Code section: module_state_clear ### */
#if CYTHON_USE_MODULE_STATE
static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) {
  __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m);
  if (!clear_module_state) return 0;
  Py_CLEAR(clear_module_state->__pyx_d);
  Py_CLEAR(clear_module_state->__pyx_b);
  Py_CLEAR(clear_module_state->__pyx_cython_runtime);
  Py_CLEAR(clear_module_state->__pyx_empty_tuple);
  Py_CLEAR(clear_module_state->__pyx_empty_bytes);
  Py_CLEAR(clear_module_state->__pyx_empty_unicode);
  #if CYTHON_PEP489_MULTI_PHASE_INIT
  __Pyx_State_RemoveModule(NULL);
  #endif
  Py_CLEAR(clear_module_state->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_PointerAttributes);
  Py_CLEAR(clear_module_state->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_MemPoolProps);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6device_Handle);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6device_Device);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_BaseMemory);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_SystemMemory);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_MemoryPool);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocatorMemory);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocator);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocator);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_8function_CPointer);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_8function_Function);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_8function_Module);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_4cuda_8function_LinkState);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_5_core_7_carray_mdspan);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_5_core_7_carray_CArray);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_5_core_7_carray_CIndexer);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_5_core_7_carray_Indexer);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_5_core_4core__ndarray_base);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal);
  Py_CLEAR(clear_module_state->__pyx_type_4cupy_3fft_9_callback__ThreadLocal);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager);
  Py_CLEAR(clear_module_state->__pyx_type_4cupy_3fft_9_callback__CallbackManager);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager);
  Py_CLEAR(clear_module_state->__pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager);
  Py_CLEAR(clear_module_state->__pyx_type_4cupy_3fft_9_callback__JITCallbackManager);
  Py_CLEAR(clear_module_state->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks);
  Py_CLEAR(clear_module_state->__pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks);
  for (int i=0; i<9; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); }
  for (int i=0; i<301; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); }
  for (int i=0; i<10; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); }
/* #### Code section: module_state_clear_contents ### */
/* #### Code section: module_state_clear_end ### */
return 0;
}
#endif
/* #### Code section: module_state_traverse ### */
#if CYTHON_USE_MODULE_STATE
static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) {
  __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m);
  if (!traverse_module_state) return 0;
  Py_VISIT(traverse_module_state->__pyx_d);
  Py_VISIT(traverse_module_state->__pyx_b);
  Py_VISIT(traverse_module_state->__pyx_cython_runtime);
  __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple);
  __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes);
  __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode);
  Py_VISIT(traverse_module_state->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_PointerAttributes);
  Py_VISIT(traverse_module_state->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_MemPoolProps);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6device_Handle);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6device_Device);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_BaseMemory);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_SystemMemory);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_MemoryPool);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocatorMemory);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocator);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocator);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_8function_CPointer);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_8function_Function);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_8function_Module);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_4cuda_8function_LinkState);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_5_core_7_carray_mdspan);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_5_core_7_carray_CArray);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_5_core_7_carray_CIndexer);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_5_core_7_carray_Indexer);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_5_core_4core__ndarray_base);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal);
  Py_VISIT(traverse_module_state->__pyx_type_4cupy_3fft_9_callback__ThreadLocal);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager);
  Py_VISIT(traverse_module_state->__pyx_type_4cupy_3fft_9_callback__CallbackManager);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager);
  Py_VISIT(traverse_module_state->__pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager);
  Py_VISIT(traverse_module_state->__pyx_type_4cupy_3fft_9_callback__JITCallbackManager);
  Py_VISIT(traverse_module_state->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks);
  Py_VISIT(traverse_module_state->__pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks);
  for (int i=0; i<9; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); }
  for (int i=0; i<301; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); }
  for (int i=0; i<10; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); }
/* #### Code section: module_state_traverse_contents ### */
/* #### Code section: module_state_traverse_end ### */
return 0;
}
#endif
/* #### Code section: module_code ### */

/* "memory.pxd":77
 *     cdef _ensure_pools_and_return_device_pool(self)
 * 
 *     cdef inline device_pool(self):             # <<<<<<<<<<<<<<
 *         # This criticial section may not be needed in practice. But without it
 *         # `self._pools` could be in an inconsistent state (the `PyObject *`
*/

static CYTHON_INLINE PyObject *__pyx_f_4cupy_4cuda_6memory_10MemoryPool_device_pool(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("device_pool", 0);

  /* "memory.pxd":82
 *         # or the tuple not fully written due to CPU/compiler optimizations).
 *         # Atomics would be another way to ensure complete safety here.
 *         with cython.critical_section(self):             # <<<<<<<<<<<<<<
 *             if self._pools is not None:
 *                 return self._pools[device.get_device_id()]
*/
  {
      __Pyx_PyCriticalSection __pyx_cs;
      __pyx_t_1 = ((PyObject *)__pyx_v_self);
      __Pyx_INCREF(__pyx_t_1);
      __Pyx_PyCriticalSection_Begin(&__pyx_cs, (PyObject*)__pyx_t_1);
      /*try:*/ {

        /* "memory.pxd":83
 *         # Atomics would be another way to ensure complete safety here.
 *         with cython.critical_section(self):
 *             if self._pools is not None:             # <<<<<<<<<<<<<<
 *                 return self._pools[device.get_device_id()]
 * 
*/
        __pyx_t_2 = (__pyx_v_self->_pools != ((PyObject*)Py_None));
        if (__pyx_t_2) {

          /* "memory.pxd":84
 *         with cython.critical_section(self):
 *             if self._pools is not None:
 *                 return self._pools[device.get_device_id()]             # <<<<<<<<<<<<<<
 * 
 *         return self._ensure_pools_and_return_device_pool()
*/
          __Pyx_XDECREF(__pyx_r);
          if (unlikely(__pyx_v_self->_pools == Py_None)) {
            PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
            __PYX_ERR(1, 84, __pyx_L4_error)
          }
          __pyx_t_3 = __pyx_f_4cupy_4cuda_6device_get_device_id(0); if (unlikely(__pyx_t_3 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 84, __pyx_L4_error)
          __pyx_t_4 = __Pyx_GetItemInt_Tuple(__pyx_v_self->_pools, __pyx_t_3, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 84, __pyx_L4_error)
          __Pyx_GOTREF(__pyx_t_4);
          __pyx_r = __pyx_t_4;
          __pyx_t_4 = 0;
          __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
          goto __pyx_L3_return;

          /* "memory.pxd":83
 *         # Atomics would be another way to ensure complete safety here.
 *         with cython.critical_section(self):
 *             if self._pools is not None:             # <<<<<<<<<<<<<<
 *                 return self._pools[device.get_device_id()]
 * 
*/
        }
      }

      /* "memory.pxd":82
 *         # or the tuple not fully written due to CPU/compiler optimizations).
 *         # Atomics would be another way to ensure complete safety here.
 *         with cython.critical_section(self):             # <<<<<<<<<<<<<<
 *             if self._pools is not None:
 *                 return self._pools[device.get_device_id()]
*/
      /*finally:*/ {
        /*normal exit:*/{
          __Pyx_PyCriticalSection_End(&__pyx_cs);
          goto __pyx_L5;
        }
        __pyx_L3_return: {
          __pyx_t_5 = __pyx_r;
          __pyx_r = 0;
          __Pyx_PyCriticalSection_End(&__pyx_cs);
          __pyx_r = __pyx_t_5;
          __pyx_t_5 = 0;
          goto __pyx_L0;
        }
        __pyx_L4_error: {
          __Pyx_PyCriticalSection_End(&__pyx_cs);
          goto __pyx_L1_error;
        }
        __pyx_L5:;
      }
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  }

  /* "memory.pxd":86
 *                 return self._pools[device.get_device_id()]
 * 
 *         return self._ensure_pools_and_return_device_pool()             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = ((struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPool *)__pyx_v_self->__pyx_vtab)->_ensure_pools_and_return_device_pool(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 86, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_r = __pyx_t_1;
  __pyx_t_1 = 0;
  goto __pyx_L0;

  /* "memory.pxd":77
 *     cdef _ensure_pools_and_return_device_pool(self)
 * 
 *     cdef inline device_pool(self):             # <<<<<<<<<<<<<<
 *         # This criticial section may not be needed in practice. But without it
 *         # `self._pools` could be in an inconsistent state (the `PyObject *`
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_AddTraceback("cupy.cuda.memory.MemoryPool.device_pool", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/_core/core.pxd":125
 * 
 * 
 * cdef inline _ndarray_base _convert_from_cupy_like(             # <<<<<<<<<<<<<<
 *         obj, error):
 *     """Inline function to convert cupy array-likes to cupy ndarray.
*/

static CYTHON_INLINE struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_f_4cupy_5_core_4core__convert_from_cupy_like(PyObject *__pyx_v_obj, PyObject *__pyx_v_error) {
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  size_t __pyx_t_4;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  PyObject *__pyx_t_8[6];
  PyObject *__pyx_t_9 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_convert_from_cupy_like", 0);

  /* "cupy/_core/core.pxd":137
 *     is not cupy-like and `None` is returned instead.
 *     """
 *     if isinstance(obj, _ndarray_base):             # <<<<<<<<<<<<<<
 *         return <_ndarray_base>obj
 *     elif hasattr(obj, "__cuda_array_interface__"):
*/
  __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_obj, __pyx_mstate_global->__pyx_ptype_4cupy_5_core_4core__ndarray_base); 
  if (__pyx_t_1) {

    /* "cupy/_core/core.pxd":138
 *     """
 *     if isinstance(obj, _ndarray_base):
 *         return <_ndarray_base>obj             # <<<<<<<<<<<<<<
 *     elif hasattr(obj, "__cuda_array_interface__"):
 *         return _convert_object_with_cuda_array_interface(obj)
*/
    __Pyx_XDECREF((PyObject *)__pyx_r);
    __Pyx_INCREF((PyObject *)((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)__pyx_v_obj));
    __pyx_r = ((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)__pyx_v_obj);
    goto __pyx_L0;

    /* "cupy/_core/core.pxd":137
 *     is not cupy-like and `None` is returned instead.
 *     """
 *     if isinstance(obj, _ndarray_base):             # <<<<<<<<<<<<<<
 *         return <_ndarray_base>obj
 *     elif hasattr(obj, "__cuda_array_interface__"):
*/
  }

  /* "cupy/_core/core.pxd":139
 *     if isinstance(obj, _ndarray_base):
 *         return <_ndarray_base>obj
 *     elif hasattr(obj, "__cuda_array_interface__"):             # <<<<<<<<<<<<<<
 *         return _convert_object_with_cuda_array_interface(obj)
 *     elif hasattr(obj, "__cupy_get_ndarray__"):
*/
  __pyx_t_1 = __Pyx_HasAttr(__pyx_v_obj, __pyx_mstate_global->__pyx_n_u_cuda_array_interface); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 139, __pyx_L1_error)
  if (__pyx_t_1) {

    /* "cupy/_core/core.pxd":140
 *         return <_ndarray_base>obj
 *     elif hasattr(obj, "__cuda_array_interface__"):
 *         return _convert_object_with_cuda_array_interface(obj)             # <<<<<<<<<<<<<<
 *     elif hasattr(obj, "__cupy_get_ndarray__"):
 *         return obj.__cupy_get_ndarray__()
*/
    __Pyx_XDECREF((PyObject *)__pyx_r);
    __pyx_t_2 = ((PyObject *)__pyx_f_4cupy_5_core_4core__convert_object_with_cuda_array_interface(__pyx_v_obj, 0)); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 140, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_r = ((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)__pyx_t_2);
    __pyx_t_2 = 0;
    goto __pyx_L0;

    /* "cupy/_core/core.pxd":139
 *     if isinstance(obj, _ndarray_base):
 *         return <_ndarray_base>obj
 *     elif hasattr(obj, "__cuda_array_interface__"):             # <<<<<<<<<<<<<<
 *         return _convert_object_with_cuda_array_interface(obj)
 *     elif hasattr(obj, "__cupy_get_ndarray__"):
*/
  }

  /* "cupy/_core/core.pxd":141
 *     elif hasattr(obj, "__cuda_array_interface__"):
 *         return _convert_object_with_cuda_array_interface(obj)
 *     elif hasattr(obj, "__cupy_get_ndarray__"):             # <<<<<<<<<<<<<<
 *         return obj.__cupy_get_ndarray__()
 * 
*/
  __pyx_t_1 = __Pyx_HasAttr(__pyx_v_obj, __pyx_mstate_global->__pyx_n_u_cupy_get_ndarray); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 141, __pyx_L1_error)
  if (__pyx_t_1) {

    /* "cupy/_core/core.pxd":142
 *         return _convert_object_with_cuda_array_interface(obj)
 *     elif hasattr(obj, "__cupy_get_ndarray__"):
 *         return obj.__cupy_get_ndarray__()             # <<<<<<<<<<<<<<
 * 
 *     if error is not False:
*/
    __Pyx_XDECREF((PyObject *)__pyx_r);
    __pyx_t_3 = __pyx_v_obj;
    __Pyx_INCREF(__pyx_t_3);
    __pyx_t_4 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_cupy_get_ndarray, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 142, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_4cupy_5_core_4core__ndarray_base))))) __PYX_ERR(2, 142, __pyx_L1_error)
    __pyx_r = ((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)__pyx_t_2);
    __pyx_t_2 = 0;
    goto __pyx_L0;

    /* "cupy/_core/core.pxd":141
 *     elif hasattr(obj, "__cuda_array_interface__"):
 *         return _convert_object_with_cuda_array_interface(obj)
 *     elif hasattr(obj, "__cupy_get_ndarray__"):             # <<<<<<<<<<<<<<
 *         return obj.__cupy_get_ndarray__()
 * 
*/
  }

  /* "cupy/_core/core.pxd":144
 *         return obj.__cupy_get_ndarray__()
 * 
 *     if error is not False:             # <<<<<<<<<<<<<<
 *         raise TypeError(
 *             f'{error} has incorrect type '
*/
  __pyx_t_1 = (__pyx_v_error != Py_False);
  if (unlikely(__pyx_t_1)) {

    /* "cupy/_core/core.pxd":145
 * 
 *     if error is not False:
 *         raise TypeError(             # <<<<<<<<<<<<<<
 *             f'{error} has incorrect type '
 *             f'(expected {_ndarray_base}, got {type(obj)}).'
*/
    __pyx_t_3 = NULL;

    /* "cupy/_core/core.pxd":146
 *     if error is not False:
 *         raise TypeError(
 *             f'{error} has incorrect type '             # <<<<<<<<<<<<<<
 *             f'(expected {_ndarray_base}, got {type(obj)}).'
 *         )
*/
    __pyx_t_5 = __Pyx_PyObject_FormatSimple(__pyx_v_error, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 146, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);

    /* "cupy/_core/core.pxd":147
 *         raise TypeError(
 *             f'{error} has incorrect type '
 *             f'(expected {_ndarray_base}, got {type(obj)}).'             # <<<<<<<<<<<<<<
 *         )
 * 
*/
    __pyx_t_6 = __Pyx_PyObject_FormatSimple(((PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_5_core_4core__ndarray_base), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 147, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
    __pyx_t_7 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_obj)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 147, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_8[0] = __pyx_t_5;
    __pyx_t_8[1] = __pyx_mstate_global->__pyx_kp_u_has_incorrect_type_expected;
    __pyx_t_8[2] = __pyx_t_6;
    __pyx_t_8[3] = __pyx_mstate_global->__pyx_kp_u_got;
    __pyx_t_8[4] = __pyx_t_7;
    __pyx_t_8[5] = __pyx_mstate_global->__pyx_kp_u_;

    /* "cupy/_core/core.pxd":146
 *     if error is not False:
 *         raise TypeError(
 *             f'{error} has incorrect type '             # <<<<<<<<<<<<<<
 *             f'(expected {_ndarray_base}, got {type(obj)}).'
 *         )
*/
    __pyx_t_9 = __Pyx_PyUnicode_Join(__pyx_t_8, 6, __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 30 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_6) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 2, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_5) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_6) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_7));
    if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 146, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __pyx_t_4 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_9};
      __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 145, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    __Pyx_Raise(__pyx_t_2, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __PYX_ERR(2, 145, __pyx_L1_error)

    /* "cupy/_core/core.pxd":144
 *         return obj.__cupy_get_ndarray__()
 * 
 *     if error is not False:             # <<<<<<<<<<<<<<
 *         raise TypeError(
 *             f'{error} has incorrect type '
*/
  }

  /* "cupy/_core/core.pxd":150
 *         )
 * 
 *     return None             # <<<<<<<<<<<<<<
*/
  __Pyx_XDECREF((PyObject *)__pyx_r);
  __pyx_r = ((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)Py_None); __Pyx_INCREF(Py_None);
  goto __pyx_L0;

  /* "cupy/_core/core.pxd":125
 * 
 * 
 * cdef inline _ndarray_base _convert_from_cupy_like(             # <<<<<<<<<<<<<<
 *         obj, error):
 *     """Inline function to convert cupy array-likes to cupy ndarray.
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_AddTraceback("cupy._core.core._convert_from_cupy_like", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF((PyObject *)__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":56
 *     cdef _CallbackManager _current_cufft_callback
 * 
 *     def __init__(self):             # <<<<<<<<<<<<<<
 *         self._current_cufft_callback = None
 * 
*/

/* Python wrapper */
static int __pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static int __pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1;
  #endif
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, __pyx_nargs); return -1; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return -1;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__init__", __pyx_kwds); return -1;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal___init__(((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static int __pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal___init__(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_self) {
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__init__", 0);

  /* "cupy/fft/_callback.pyx":57
 * 
 *     def __init__(self):
 *         self._current_cufft_callback = None             # <<<<<<<<<<<<<<
 * 
 *     @staticmethod
*/
  __Pyx_INCREF(Py_None);
  __Pyx_GIVEREF(Py_None);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->_current_cufft_callback);
  __Pyx_DECREF((PyObject *)__pyx_v_self->_current_cufft_callback);
  __pyx_v_self->_current_cufft_callback = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None);

  /* "cupy/fft/_callback.pyx":56
 *     cdef _CallbackManager _current_cufft_callback
 * 
 *     def __init__(self):             # <<<<<<<<<<<<<<
 *         self._current_cufft_callback = None
 * 
*/

  /* function exit code */
  __pyx_r = 0;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":59
 *         self._current_cufft_callback = None
 * 
 *     @staticmethod             # <<<<<<<<<<<<<<
 *     cdef _ThreadLocal get():
 *         cdef _ThreadLocal tls
*/

static struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_f_4cupy_3fft_9_callback_12_ThreadLocal_get(void) {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_tls = 0;
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  int __pyx_t_5;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  PyObject *__pyx_t_8 = NULL;
  PyObject *__pyx_t_9 = NULL;
  size_t __pyx_t_10;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("get", 0);

  /* "cupy/fft/_callback.pyx":62
 *     cdef _ThreadLocal get():
 *         cdef _ThreadLocal tls
 *         try:             # <<<<<<<<<<<<<<
 *             tls = _callback_thread_local.tls
 *         except AttributeError:
*/
  {
    __Pyx_PyThreadState_declare
    __Pyx_PyThreadState_assign
    __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3);
    __Pyx_XGOTREF(__pyx_t_1);
    __Pyx_XGOTREF(__pyx_t_2);
    __Pyx_XGOTREF(__pyx_t_3);
    /*try:*/ {

      /* "cupy/fft/_callback.pyx":63
 *         cdef _ThreadLocal tls
 *         try:
 *             tls = _callback_thread_local.tls             # <<<<<<<<<<<<<<
 *         except AttributeError:
 *             tls = _callback_thread_local.tls = _ThreadLocal()
*/
      __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_4cupy_3fft_9_callback__callback_thread_local, __pyx_mstate_global->__pyx_n_u_tls); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 63, __pyx_L3_error)
      __Pyx_GOTREF(__pyx_t_4);
      if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal))))) __PYX_ERR(0, 63, __pyx_L3_error)
      __pyx_v_tls = ((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_t_4);
      __pyx_t_4 = 0;

      /* "cupy/fft/_callback.pyx":62
 *     cdef _ThreadLocal get():
 *         cdef _ThreadLocal tls
 *         try:             # <<<<<<<<<<<<<<
 *             tls = _callback_thread_local.tls
 *         except AttributeError:
*/
    }
    __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    goto __pyx_L8_try_end;
    __pyx_L3_error:;
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;

    /* "cupy/fft/_callback.pyx":64
 *         try:
 *             tls = _callback_thread_local.tls
 *         except AttributeError:             # <<<<<<<<<<<<<<
 *             tls = _callback_thread_local.tls = _ThreadLocal()
 *         return tls
*/
    __pyx_t_5 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_AttributeError))));
    if (__pyx_t_5) {
      __Pyx_AddTraceback("cupy.fft._callback._ThreadLocal.get", __pyx_clineno, __pyx_lineno, __pyx_filename);
      if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(0, 64, __pyx_L5_except_error)
      __Pyx_XGOTREF(__pyx_t_4);
      __Pyx_XGOTREF(__pyx_t_6);
      __Pyx_XGOTREF(__pyx_t_7);

      /* "cupy/fft/_callback.pyx":65
 *             tls = _callback_thread_local.tls
 *         except AttributeError:
 *             tls = _callback_thread_local.tls = _ThreadLocal()             # <<<<<<<<<<<<<<
 *         return tls
 * 
*/
      __pyx_t_9 = NULL;
      __pyx_t_10 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_9, NULL};
        __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal, __pyx_callargs+__pyx_t_10, (1-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0;
        if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 65, __pyx_L5_except_error)
        __Pyx_GOTREF((PyObject *)__pyx_t_8);
      }
      __Pyx_INCREF(__pyx_t_8);
      __Pyx_XDECREF_SET(__pyx_v_tls, ((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_t_8));
      if (__Pyx_PyObject_SetAttrStr(__pyx_v_4cupy_3fft_9_callback__callback_thread_local, __pyx_mstate_global->__pyx_n_u_tls, __pyx_t_8) < (0)) __PYX_ERR(0, 65, __pyx_L5_except_error)
      __Pyx_DECREF((PyObject *)__pyx_t_8); __pyx_t_8 = 0;
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
      __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
      goto __pyx_L4_exception_handled;
    }
    goto __pyx_L5_except_error;

    /* "cupy/fft/_callback.pyx":62
 *     cdef _ThreadLocal get():
 *         cdef _ThreadLocal tls
 *         try:             # <<<<<<<<<<<<<<
 *             tls = _callback_thread_local.tls
 *         except AttributeError:
*/
    __pyx_L5_except_error:;
    __Pyx_XGIVEREF(__pyx_t_1);
    __Pyx_XGIVEREF(__pyx_t_2);
    __Pyx_XGIVEREF(__pyx_t_3);
    __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3);
    goto __pyx_L1_error;
    __pyx_L4_exception_handled:;
    __Pyx_XGIVEREF(__pyx_t_1);
    __Pyx_XGIVEREF(__pyx_t_2);
    __Pyx_XGIVEREF(__pyx_t_3);
    __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3);
    __pyx_L8_try_end:;
  }

  /* "cupy/fft/_callback.pyx":66
 *         except AttributeError:
 *             tls = _callback_thread_local.tls = _ThreadLocal()
 *         return tls             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_XDECREF((PyObject *)__pyx_r);
  __Pyx_INCREF((PyObject *)__pyx_v_tls);
  __pyx_r = __pyx_v_tls;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":59
 *         self._current_cufft_callback = None
 * 
 *     @staticmethod             # <<<<<<<<<<<<<<
 *     cdef _ThreadLocal get():
 *         cdef _ThreadLocal tls
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_AddTraceback("cupy.fft._callback._ThreadLocal.get", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF((PyObject *)__pyx_v_tls);
  __Pyx_XGIVEREF((PyObject *)__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_3__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_12_ThreadLocal_2__reduce_cython__, "_ThreadLocal.__reduce_cython__(self)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_3__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return NULL;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal_2__reduce_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal_2__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_self) {
  PyObject *__pyx_v_state = 0;
  PyObject *__pyx_v__dict = 0;
  int __pyx_v_use_setstate;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__reduce_cython__", 0);

  /* "(tree fragment)":5
 *     cdef object _dict
 *     cdef bint use_setstate
 *     state = (self._current_cufft_callback,)             # <<<<<<<<<<<<<<
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
*/
  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_INCREF((PyObject *)__pyx_v_self->_current_cufft_callback);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->_current_cufft_callback);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self->_current_cufft_callback)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __pyx_v_state = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":6
 *     cdef bint use_setstate
 *     state = (self._current_cufft_callback,)
 *     _dict = getattr(self, '__dict__', None)             # <<<<<<<<<<<<<<
 *     if _dict is not None and _dict:
 *         state += (_dict,)
*/
  __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 6, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v__dict = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "(tree fragment)":7
 *     state = (self._current_cufft_callback,)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
  __pyx_t_3 = (__pyx_v__dict != Py_None);
  if (__pyx_t_3) {
  } else {
    __pyx_t_2 = __pyx_t_3;
    goto __pyx_L4_bool_binop_done;
  }
  __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(3, 7, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_3;
  __pyx_L4_bool_binop_done:;
  if (__pyx_t_2) {

    /* "(tree fragment)":8
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
 *         state += (_dict,)             # <<<<<<<<<<<<<<
 *         use_setstate = True
 *     else:
*/
    __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(__pyx_v__dict);
    __Pyx_GIVEREF(__pyx_v__dict);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(3, 8, __pyx_L1_error);
    __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "(tree fragment)":9
 *     if _dict is not None and _dict:
 *         state += (_dict,)
 *         use_setstate = True             # <<<<<<<<<<<<<<
 *     else:
 *         use_setstate = self._current_cufft_callback is not None
*/
    __pyx_v_use_setstate = 1;

    /* "(tree fragment)":7
 *     state = (self._current_cufft_callback,)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
    goto __pyx_L3;
  }

  /* "(tree fragment)":11
 *         use_setstate = True
 *     else:
 *         use_setstate = self._current_cufft_callback is not None             # <<<<<<<<<<<<<<
 *     if use_setstate:
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, None), state
*/
  /*else*/ {
    __pyx_t_2 = (((PyObject *)__pyx_v_self->_current_cufft_callback) != Py_None);
    __pyx_v_use_setstate = __pyx_t_2;
  }
  __pyx_L3:;

  /* "(tree fragment)":12
 *     else:
 *         use_setstate = self._current_cufft_callback is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, None), state
 *     else:
*/
  if (__pyx_v_use_setstate) {

    /* "(tree fragment)":13
 *         use_setstate = self._current_cufft_callback is not None
 *     if use_setstate:
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, None), state             # <<<<<<<<<<<<<<
 *     else:
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, state)
*/
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__ThreadLocal); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_146004246);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_146004246);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_146004246) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(Py_None);
    __Pyx_GIVEREF(Py_None);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_GIVEREF(__pyx_t_4);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_4 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_5;
    __pyx_t_5 = 0;
    goto __pyx_L0;

    /* "(tree fragment)":12
 *     else:
 *         use_setstate = self._current_cufft_callback is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, None), state
 *     else:
*/
  }

  /* "(tree fragment)":15
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, None), state
 *     else:
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, state)             # <<<<<<<<<<<<<<
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__ThreadLocal__set_state(self, __pyx_state)
*/
  /*else*/ {
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__ThreadLocal); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_146004246);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_146004246);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_146004246) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_GIVEREF(__pyx_t_5);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_5 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_4;
    __pyx_t_4 = 0;
    goto __pyx_L0;
  }

  /* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback._ThreadLocal.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_state);
  __Pyx_XDECREF(__pyx_v__dict);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__ThreadLocal__set_state(self, __pyx_state)
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_5__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_12_ThreadLocal_4__setstate_cython__, "_ThreadLocal.__setstate_cython__(self, __pyx_state)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_5__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 16, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(3, 16, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(3, 16, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
    }
    __pyx_v___pyx_state = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(3, 16, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._ThreadLocal.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal_4__setstate_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_v_self), __pyx_v___pyx_state);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_12_ThreadLocal_4__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_self, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__setstate_cython__", 0);

  /* "(tree fragment)":17
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, state)
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__ThreadLocal__set_state(self, __pyx_state)             # <<<<<<<<<<<<<<
*/
  __pyx_t_1 = __pyx_v___pyx_state;
  __Pyx_INCREF(__pyx_t_1);
  if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(3, 17, __pyx_L1_error)
  if (unlikely(__pyx_t_1 == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
    __PYX_ERR(3, 17, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__ThreadLocal__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 17, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__ThreadLocal, (type(self), 0x8b3d916, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__ThreadLocal__set_state(self, __pyx_state)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_AddTraceback("cupy.fft._callback._ThreadLocal.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":69
 * 
 * 
 * cdef inline void _set_vars() except*:             # <<<<<<<<<<<<<<
 *     from cupy.cuda.cufft import getVersion as get_cufft_version
 * 
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__set_vars(void) {
  PyObject *__pyx_v_get_cufft_version = NULL;
  PyObject *__pyx_v_cxx = 0;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  Py_ssize_t __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  size_t __pyx_t_7;
  int __pyx_t_8;
  PyObject *__pyx_t_9 = NULL;
  PyObject *__pyx_t_10 = NULL;
  int __pyx_t_11;
  int __pyx_t_12;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_set_vars", 0);

  /* "cupy/fft/_callback.pyx":70
 * 
 * cdef inline void _set_vars() except*:
 *     from cupy.cuda.cufft import getVersion as get_cufft_version             # <<<<<<<<<<<<<<
 * 
 *     global _cc, _python_include, _cuda_path, _cuda_include, _nvprune
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_getVersion};
    __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda_cufft, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 70, __pyx_L1_error)
  }
  __pyx_t_1 = __pyx_t_2;
  __Pyx_GOTREF(__pyx_t_1);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_getVersion};
    __pyx_t_3 = 0; {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_1, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 70, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      switch (__pyx_t_3) {
        case 0:
        __Pyx_INCREF(__pyx_t_4);
        __pyx_v_get_cufft_version = __pyx_t_4;
        break;
        default:;
      }
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":76
 *     global _cc_major_map
 * 
 *     cdef str cxx = sysconfig.get_config_var('CXX')             # <<<<<<<<<<<<<<
 *     if cxx is not None:
 *         _cc = cxx.split(' ')
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_sysconfig); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 76, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_get_config_var); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 76, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_6))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_6, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_CXX};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 76, __pyx_L1_error)
  __pyx_v_cxx = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":77
 * 
 *     cdef str cxx = sysconfig.get_config_var('CXX')
 *     if cxx is not None:             # <<<<<<<<<<<<<<
 *         _cc = cxx.split(' ')
 *     elif 'CXX' in os.environ:
*/
  __pyx_t_8 = (__pyx_v_cxx != ((PyObject*)Py_None));
  if (__pyx_t_8) {

    /* "cupy/fft/_callback.pyx":78
 *     cdef str cxx = sysconfig.get_config_var('CXX')
 *     if cxx is not None:
 *         _cc = cxx.split(' ')             # <<<<<<<<<<<<<<
 *     elif 'CXX' in os.environ:
 *         _cc = os.environ.get('CXX').split(' ')
*/
    if (unlikely(__pyx_v_cxx == Py_None)) {
      PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "split");
      __PYX_ERR(0, 78, __pyx_L1_error)
    }
    __pyx_t_1 = PyUnicode_Split(__pyx_v_cxx, __pyx_mstate_global->__pyx_kp_u__3, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cc);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cc, ((PyObject*)__pyx_t_1));
    __Pyx_GIVEREF(__pyx_t_1);
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":77
 * 
 *     cdef str cxx = sysconfig.get_config_var('CXX')
 *     if cxx is not None:             # <<<<<<<<<<<<<<
 *         _cc = cxx.split(' ')
 *     elif 'CXX' in os.environ:
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":79
 *     if cxx is not None:
 *         _cc = cxx.split(' ')
 *     elif 'CXX' in os.environ:             # <<<<<<<<<<<<<<
 *         _cc = os.environ.get('CXX').split(' ')
 *     else:
*/
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 79, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 79, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_8 = (__Pyx_PySequence_ContainsTF(__pyx_mstate_global->__pyx_n_u_CXX, __pyx_t_6, Py_EQ)); if (unlikely((__pyx_t_8 < 0))) __PYX_ERR(0, 79, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  if (__pyx_t_8) {

    /* "cupy/fft/_callback.pyx":80
 *         _cc = cxx.split(' ')
 *     elif 'CXX' in os.environ:
 *         _cc = os.environ.get('CXX').split(' ')             # <<<<<<<<<<<<<<
 *     else:
 *         _cc = None
*/
    __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 80, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
    __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 80, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_10);
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    __pyx_t_5 = __pyx_t_10;
    __Pyx_INCREF(__pyx_t_5);
    __pyx_t_7 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_n_u_CXX};
      __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 80, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __pyx_t_1 = __pyx_t_4;
    __Pyx_INCREF(__pyx_t_1);
    __pyx_t_7 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_kp_u__3};
      __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_split, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 80, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_6);
    }
    if (!(likely(PyList_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("list", __pyx_t_6))) __PYX_ERR(0, 80, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cc);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cc, ((PyObject*)__pyx_t_6));
    __Pyx_GIVEREF(__pyx_t_6);
    __pyx_t_6 = 0;

    /* "cupy/fft/_callback.pyx":79
 *     if cxx is not None:
 *         _cc = cxx.split(' ')
 *     elif 'CXX' in os.environ:             # <<<<<<<<<<<<<<
 *         _cc = os.environ.get('CXX').split(' ')
 *     else:
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":82
 *         _cc = os.environ.get('CXX').split(' ')
 *     else:
 *         _cc = None             # <<<<<<<<<<<<<<
 * 
 *     _python_include = sysconfig.get_path('include')
*/
  /*else*/ {
    __Pyx_INCREF(Py_None);
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cc);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cc, ((PyObject*)Py_None));
    __Pyx_GIVEREF(Py_None);
  }
  __pyx_L3:;

  /* "cupy/fft/_callback.pyx":84
 *         _cc = None
 * 
 *     _python_include = sysconfig.get_path('include')             # <<<<<<<<<<<<<<
 * 
 *     _cuda_path = get_cuda_path()
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_sysconfig); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 84, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_get_path); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 84, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_10);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_10))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_10);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_10, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_include};
    __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
    if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 84, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_6))) __PYX_ERR(0, 84, __pyx_L1_error)
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__python_include);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__python_include, ((PyObject*)__pyx_t_6));
  __Pyx_GIVEREF(__pyx_t_6);
  __pyx_t_6 = 0;

  /* "cupy/fft/_callback.pyx":86
 *     _python_include = sysconfig.get_path('include')
 * 
 *     _cuda_path = get_cuda_path()             # <<<<<<<<<<<<<<
 *     if _cuda_path is not None:
 *         _nvprune = find_nvidia_binary_utility('nvprune')
*/
  __pyx_t_10 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_get_cuda_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 86, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_4))) {
    __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_4);
    assert(__pyx_t_10);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
    __Pyx_INCREF(__pyx_t_10);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_10, NULL};
    __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_7, (1-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 86, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_6))) __PYX_ERR(0, 86, __pyx_L1_error)
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cuda_path);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cuda_path, ((PyObject*)__pyx_t_6));
  __Pyx_GIVEREF(__pyx_t_6);
  __pyx_t_6 = 0;

  /* "cupy/fft/_callback.pyx":87
 * 
 *     _cuda_path = get_cuda_path()
 *     if _cuda_path is not None:             # <<<<<<<<<<<<<<
 *         _nvprune = find_nvidia_binary_utility('nvprune')
 *         if _nvprune is None:
*/
  __pyx_t_8 = (__pyx_v_4cupy_3fft_9_callback__cuda_path != ((PyObject*)Py_None));
  if (__pyx_t_8) {

    /* "cupy/fft/_callback.pyx":88
 *     _cuda_path = get_cuda_path()
 *     if _cuda_path is not None:
 *         _nvprune = find_nvidia_binary_utility('nvprune')             # <<<<<<<<<<<<<<
 *         if _nvprune is None:
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
*/
    __pyx_t_4 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_find_nvidia_binary_utility); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 88, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_10);
    __pyx_t_7 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_10))) {
      __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_10);
      assert(__pyx_t_4);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10);
      __Pyx_INCREF(__pyx_t_4);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_10, __pyx__function);
      __pyx_t_7 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_nvprune};
      __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
      if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 88, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_6);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_6))) __PYX_ERR(0, 88, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvprune, ((PyObject*)__pyx_t_6));
    __Pyx_GIVEREF(__pyx_t_6);
    __pyx_t_6 = 0;

    /* "cupy/fft/_callback.pyx":89
 *     if _cuda_path is not None:
 *         _nvprune = find_nvidia_binary_utility('nvprune')
 *         if _nvprune is None:             # <<<<<<<<<<<<<<
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
 *             if not os.path.isfile(_nvprune):
*/
    __pyx_t_8 = (__pyx_v_4cupy_3fft_9_callback__nvprune == ((PyObject*)Py_None));
    if (__pyx_t_8) {

      /* "cupy/fft/_callback.pyx":90
 *         _nvprune = find_nvidia_binary_utility('nvprune')
 *         if _nvprune is None:
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')             # <<<<<<<<<<<<<<
 *             if not os.path.isfile(_nvprune):
 *                 _nvprune = None
*/
      __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 90, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 90, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      __pyx_t_10 = __pyx_t_1;
      __Pyx_INCREF(__pyx_t_10);
      __pyx_t_7 = 0;
      {
        PyObject *__pyx_callargs[3] = {__pyx_t_10, __pyx_v_4cupy_3fft_9_callback__cuda_path, __pyx_mstate_global->__pyx_kp_u_bin_nvprune};
        __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
        __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
        if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 90, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_6);
      }
      if (!(likely(PyUnicode_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_6))) __PYX_ERR(0, 90, __pyx_L1_error)
      __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
      __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvprune, ((PyObject*)__pyx_t_6));
      __Pyx_GIVEREF(__pyx_t_6);
      __pyx_t_6 = 0;

      /* "cupy/fft/_callback.pyx":91
 *         if _nvprune is None:
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
 *             if not os.path.isfile(_nvprune):             # <<<<<<<<<<<<<<
 *                 _nvprune = None
 *     else:
*/
      __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 91, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_10);
      __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 91, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
      __pyx_t_1 = __pyx_t_4;
      __Pyx_INCREF(__pyx_t_1);
      __pyx_t_7 = 0;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_v_4cupy_3fft_9_callback__nvprune};
        __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_isfile, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 91, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_6);
      }
      __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely((__pyx_t_8 < 0))) __PYX_ERR(0, 91, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      __pyx_t_11 = (!__pyx_t_8);
      if (__pyx_t_11) {

        /* "cupy/fft/_callback.pyx":92
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
 *             if not os.path.isfile(_nvprune):
 *                 _nvprune = None             # <<<<<<<<<<<<<<
 *     else:
 *         # _prune() needs both nvprune and _cuda_path (for
*/
        __Pyx_INCREF(Py_None);
        __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
        __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvprune, ((PyObject*)Py_None));
        __Pyx_GIVEREF(Py_None);

        /* "cupy/fft/_callback.pyx":91
 *         if _nvprune is None:
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
 *             if not os.path.isfile(_nvprune):             # <<<<<<<<<<<<<<
 *                 _nvprune = None
 *     else:
*/
      }

      /* "cupy/fft/_callback.pyx":89
 *     if _cuda_path is not None:
 *         _nvprune = find_nvidia_binary_utility('nvprune')
 *         if _nvprune is None:             # <<<<<<<<<<<<<<
 *             _nvprune = os.path.join(_cuda_path, 'bin/nvprune')
 *             if not os.path.isfile(_nvprune):
*/
    }

    /* "cupy/fft/_callback.pyx":87
 * 
 *     _cuda_path = get_cuda_path()
 *     if _cuda_path is not None:             # <<<<<<<<<<<<<<
 *         _nvprune = find_nvidia_binary_utility('nvprune')
 *         if _nvprune is None:
*/
    goto __pyx_L4;
  }

  /* "cupy/fft/_callback.pyx":96
 *         # _prune() needs both nvprune and _cuda_path (for
 *         # libcufft_static.a), so skip nvprune if there is no root.
 *         _nvprune = None             # <<<<<<<<<<<<<<
 * 
 *     _cuda_include = find_nvidia_header_directory('cudart')
*/
  /*else*/ {
    __Pyx_INCREF(Py_None);
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvprune, ((PyObject*)Py_None));
    __Pyx_GIVEREF(Py_None);
  }
  __pyx_L4:;

  /* "cupy/fft/_callback.pyx":98
 *         _nvprune = None
 * 
 *     _cuda_include = find_nvidia_header_directory('cudart')             # <<<<<<<<<<<<<<
 * 
 *     _build_ver = str(runtime.runtimeGetVersion())
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_find_nvidia_header_directory); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 98, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_1))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_1);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_1);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_1, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_cudart};
    __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_1, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 98, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_6))) __PYX_ERR(0, 98, __pyx_L1_error)
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cuda_include);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cuda_include, ((PyObject*)__pyx_t_6));
  __Pyx_GIVEREF(__pyx_t_6);
  __pyx_t_6 = 0;

  /* "cupy/fft/_callback.pyx":100
 *     _cuda_include = find_nvidia_header_directory('cudart')
 * 
 *     _build_ver = str(runtime.runtimeGetVersion())             # <<<<<<<<<<<<<<
 *     _cufft_ver = get_cufft_version()
 *     # Many archs are removed since CUDA 13.0.0 (cuFFT 12.0.0).
*/
  __pyx_t_12 = __pyx_f_13cupy_backends_4cuda_3api_7runtime_runtimeGetVersion(0); if (unlikely(__pyx_t_12 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 100, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyLong_From_int(__pyx_t_12); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 100, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_1 = __Pyx_PyObject_Unicode(__pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 100, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__build_ver);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__build_ver, ((PyObject*)__pyx_t_1));
  __Pyx_GIVEREF(__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":101
 * 
 *     _build_ver = str(runtime.runtimeGetVersion())
 *     _cufft_ver = get_cufft_version()             # <<<<<<<<<<<<<<
 *     # Many archs are removed since CUDA 13.0.0 (cuFFT 12.0.0).
 *     if _cufft_ver < 12000:
*/
  __pyx_t_6 = NULL;
  __Pyx_INCREF(__pyx_v_get_cufft_version);
  __pyx_t_4 = __pyx_v_get_cufft_version; 
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_4))) {
    __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4);
    assert(__pyx_t_6);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
    __Pyx_INCREF(__pyx_t_6);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_6, NULL};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_7, (1-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 101, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __pyx_t_12 = __Pyx_PyLong_As_int(__pyx_t_1); if (unlikely((__pyx_t_12 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 101, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_v_4cupy_3fft_9_callback__cufft_ver = __pyx_t_12;

  /* "cupy/fft/_callback.pyx":103
 *     _cufft_ver = get_cufft_version()
 *     # Many archs are removed since CUDA 13.0.0 (cuFFT 12.0.0).
 *     if _cufft_ver < 12000:             # <<<<<<<<<<<<<<
 *         _cc_major_map = {**_cc_major_map, **{
 *             '7': ('70', '72', '75'),
*/
  __pyx_t_11 = (__pyx_v_4cupy_3fft_9_callback__cufft_ver < 0x2EE0);
  if (__pyx_t_11) {

    /* "cupy/fft/_callback.pyx":104
 *     # Many archs are removed since CUDA 13.0.0 (cuFFT 12.0.0).
 *     if _cufft_ver < 12000:
 *         _cc_major_map = {**_cc_major_map, **{             # <<<<<<<<<<<<<<
 *             '7': ('70', '72', '75'),
 *             '6': ('60', '61', '62'),
*/
    if (unlikely(__pyx_v_4cupy_3fft_9_callback__cc_major_map == Py_None)) {
      PyErr_SetString(PyExc_TypeError, "argument after ** must be a mapping, not NoneType");
      __PYX_ERR(0, 104, __pyx_L1_error)
    }
    __pyx_t_1 = PyDict_Copy(__pyx_v_4cupy_3fft_9_callback__cc_major_map); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 104, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);

    /* "cupy/fft/_callback.pyx":105
 *     if _cufft_ver < 12000:
 *         _cc_major_map = {**_cc_major_map, **{
 *             '7': ('70', '72', '75'),             # <<<<<<<<<<<<<<
 *             '6': ('60', '61', '62'),
 *             '5': ('50', '52', '53'),
*/
    if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_7, __pyx_mstate_global->__pyx_tuple[0]) < (0)) __PYX_ERR(0, 105, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":106
 *         _cc_major_map = {**_cc_major_map, **{
 *             '7': ('70', '72', '75'),
 *             '6': ('60', '61', '62'),             # <<<<<<<<<<<<<<
 *             '5': ('50', '52', '53'),
 *         }}
*/
    if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_6, __pyx_mstate_global->__pyx_tuple[1]) < (0)) __PYX_ERR(0, 106, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":107
 *             '7': ('70', '72', '75'),
 *             '6': ('60', '61', '62'),
 *             '5': ('50', '52', '53'),             # <<<<<<<<<<<<<<
 *         }}
 *     _ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
*/
    if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_5, __pyx_mstate_global->__pyx_tuple[2]) < (0)) __PYX_ERR(0, 107, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cc_major_map);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cc_major_map, ((PyObject*)__pyx_t_1));
    __Pyx_GIVEREF(__pyx_t_1);
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":103
 *     _cufft_ver = get_cufft_version()
 *     # Many archs are removed since CUDA 13.0.0 (cuFFT 12.0.0).
 *     if _cufft_ver < 12000:             # <<<<<<<<<<<<<<
 *         _cc_major_map = {**_cc_major_map, **{
 *             '7': ('70', '72', '75'),
*/
  }

  /* "cupy/fft/_callback.pyx":109
 *             '5': ('50', '52', '53'),
 *         }}
 *     _ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')             # <<<<<<<<<<<<<<
 * 
 *     _set_cupy_paths()
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_sysconfig); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 109, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_get_config_var); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 109, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_10);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_10))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_10);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_10, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_EXT_SUFFIX};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 109, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 109, __pyx_L1_error)
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__ext_suffix);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__ext_suffix, ((PyObject*)__pyx_t_1));
  __Pyx_GIVEREF(__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":111
 *     _ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
 * 
 *     _set_cupy_paths()             # <<<<<<<<<<<<<<
 *     _set_nvcc_path()
 *     _is_init = True
*/
  __pyx_f_4cupy_3fft_9_callback__set_cupy_paths(); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 111, __pyx_L1_error)

  /* "cupy/fft/_callback.pyx":112
 * 
 *     _set_cupy_paths()
 *     _set_nvcc_path()             # <<<<<<<<<<<<<<
 *     _is_init = True
 * 
*/
  __pyx_f_4cupy_3fft_9_callback__set_nvcc_path(); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 112, __pyx_L1_error)

  /* "cupy/fft/_callback.pyx":113
 *     _set_cupy_paths()
 *     _set_nvcc_path()
 *     _is_init = True             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __pyx_v_4cupy_3fft_9_callback__is_init = 1;

  /* "cupy/fft/_callback.pyx":69
 * 
 * 
 * cdef inline void _set_vars() except*:             # <<<<<<<<<<<<<<
 *     from cupy.cuda.cufft import getVersion as get_cufft_version
 * 
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_XDECREF(__pyx_t_10);
  __Pyx_AddTraceback("cupy.fft._callback._set_vars", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_get_cufft_version);
  __Pyx_XDECREF(__pyx_v_cxx);
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":116
 * 
 * 
 * cdef inline void _set_cupy_paths() except*:             # <<<<<<<<<<<<<<
 *     # Workaround: older Cython cannot use __file__ in global scope
 *     global _cupy_root, _cupy_include, _source_dir
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__set_cupy_paths(void) {
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  PyObject *__pyx_t_8 = NULL;
  size_t __pyx_t_9;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_set_cupy_paths", 0);

  /* "cupy/fft/_callback.pyx":119
 *     # Workaround: older Cython cannot use __file__ in global scope
 *     global _cupy_root, _cupy_include, _source_dir
 *     if _cupy_root is None:             # <<<<<<<<<<<<<<
 *         _cupy_root = os.path.join(os.path.dirname(__file__), '..')
 *         _cupy_include = os.path.join(_cupy_root, '_core/include')
*/
  __pyx_t_1 = (__pyx_v_4cupy_3fft_9_callback__cupy_root == ((PyObject*)Py_None));
  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":120
 *     global _cupy_root, _cupy_include, _source_dir
 *     if _cupy_root is None:
 *         _cupy_root = os.path.join(os.path.dirname(__file__), '..')             # <<<<<<<<<<<<<<
 *         _cupy_include = os.path.join(_cupy_root, '_core/include')
 *         _source_dir = os.path.join(_cupy_root, 'cuda')
*/
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 120, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __pyx_t_3 = __pyx_t_5;
    __Pyx_INCREF(__pyx_t_3);
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 120, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 120, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_8);
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __pyx_t_6 = __pyx_t_8;
    __Pyx_INCREF(__pyx_t_6);
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_file); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 120, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_7};
      __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_dirname, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
      __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_4, __pyx_mstate_global->__pyx_kp_u__4};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 120, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 120, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cupy_root);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cupy_root, ((PyObject*)__pyx_t_2));
    __Pyx_GIVEREF(__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":121
 *     if _cupy_root is None:
 *         _cupy_root = os.path.join(os.path.dirname(__file__), '..')
 *         _cupy_include = os.path.join(_cupy_root, '_core/include')             # <<<<<<<<<<<<<<
 *         _source_dir = os.path.join(_cupy_root, 'cuda')
 * 
*/
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 121, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 121, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __pyx_t_5 = __pyx_t_3;
    __Pyx_INCREF(__pyx_t_5);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_4cupy_3fft_9_callback__cupy_root, __pyx_mstate_global->__pyx_kp_u_core_include};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 121, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 121, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cupy_include);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cupy_include, ((PyObject*)__pyx_t_2));
    __Pyx_GIVEREF(__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":122
 *         _cupy_root = os.path.join(os.path.dirname(__file__), '..')
 *         _cupy_include = os.path.join(_cupy_root, '_core/include')
 *         _source_dir = os.path.join(_cupy_root, 'cuda')             # <<<<<<<<<<<<<<
 * 
 * 
*/
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 122, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 122, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __pyx_t_3 = __pyx_t_4;
    __Pyx_INCREF(__pyx_t_3);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_v_4cupy_3fft_9_callback__cupy_root, __pyx_mstate_global->__pyx_n_u_cuda};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 122, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 122, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__source_dir);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__source_dir, ((PyObject*)__pyx_t_2));
    __Pyx_GIVEREF(__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":119
 *     # Workaround: older Cython cannot use __file__ in global scope
 *     global _cupy_root, _cupy_include, _source_dir
 *     if _cupy_root is None:             # <<<<<<<<<<<<<<
 *         _cupy_root = os.path.join(os.path.dirname(__file__), '..')
 *         _cupy_include = os.path.join(_cupy_root, '_core/include')
*/
  }

  /* "cupy/fft/_callback.pyx":116
 * 
 * 
 * cdef inline void _set_cupy_paths() except*:             # <<<<<<<<<<<<<<
 *     # Workaround: older Cython cannot use __file__ in global scope
 *     global _cupy_root, _cupy_include, _source_dir
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_AddTraceback("cupy.fft._callback._set_cupy_paths", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":125
 * 
 * 
 * cdef inline void _set_nvcc_path() except*:             # <<<<<<<<<<<<<<
 *     # get_nvcc_path() could be a long string like "ccache nvcc ..." from NVCC
 *     cdef str nvcc = None
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__set_nvcc_path(void) {
  PyObject *__pyx_v_nvcc = 0;
  PyObject *__pyx_v_f = 0;
  int __pyx_v_i;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  int __pyx_t_6;
  Py_ssize_t __pyx_t_7;
  PyObject *__pyx_t_8 = NULL;
  int __pyx_t_9;
  long __pyx_t_10;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_set_nvcc_path", 0);

  /* "cupy/fft/_callback.pyx":127
 * cdef inline void _set_nvcc_path() except*:
 *     # get_nvcc_path() could be a long string like "ccache nvcc ..." from NVCC
 *     cdef str nvcc = None             # <<<<<<<<<<<<<<
 *     cdef str f
 *     cdef int i
*/
  __Pyx_INCREF(Py_None);
  __pyx_v_nvcc = ((PyObject*)Py_None);

  /* "cupy/fft/_callback.pyx":132
 * 
 *     global _nvcc
 *     if _nvcc == []:             # <<<<<<<<<<<<<<
 *         nvcc = get_nvcc_path()
 *         if nvcc is not None:
*/
  __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 132, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_2 = PyObject_RichCompare(__pyx_v_4cupy_3fft_9_callback__nvcc, __pyx_t_1, Py_EQ); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 132, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 132, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  if (__pyx_t_3) {

    /* "cupy/fft/_callback.pyx":133
 *     global _nvcc
 *     if _nvcc == []:
 *         nvcc = get_nvcc_path()             # <<<<<<<<<<<<<<
 *         if nvcc is not None:
 *             _nvcc = nvcc.split(' ')
*/
    __pyx_t_1 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_get_nvcc_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 133, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_5 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_4))) {
      __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_4);
      assert(__pyx_t_1);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
      __Pyx_INCREF(__pyx_t_1);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
      __pyx_t_5 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_1, NULL};
      __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 133, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 133, __pyx_L1_error)
    __Pyx_DECREF_SET(__pyx_v_nvcc, ((PyObject*)__pyx_t_2));
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":134
 *     if _nvcc == []:
 *         nvcc = get_nvcc_path()
 *         if nvcc is not None:             # <<<<<<<<<<<<<<
 *             _nvcc = nvcc.split(' ')
 *             # if the host compiler is set in NVCC, we force it to align with
*/
    __pyx_t_3 = (__pyx_v_nvcc != ((PyObject*)Py_None));
    if (__pyx_t_3) {

      /* "cupy/fft/_callback.pyx":135
 *         nvcc = get_nvcc_path()
 *         if nvcc is not None:
 *             _nvcc = nvcc.split(' ')             # <<<<<<<<<<<<<<
 *             # if the host compiler is set in NVCC, we force it to align with
 *             # the one reported by sysconfig
*/
      if (unlikely(__pyx_v_nvcc == Py_None)) {
        PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "split");
        __PYX_ERR(0, 135, __pyx_L1_error)
      }
      __pyx_t_2 = PyUnicode_Split(__pyx_v_nvcc, __pyx_mstate_global->__pyx_kp_u__3, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 135, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
      __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvcc);
      __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvcc, ((PyObject*)__pyx_t_2));
      __Pyx_GIVEREF(__pyx_t_2);
      __pyx_t_2 = 0;

      /* "cupy/fft/_callback.pyx":138
 *             # if the host compiler is set in NVCC, we force it to align with
 *             # the one reported by sysconfig
 *             for i, f in enumerate(_nvcc):             # <<<<<<<<<<<<<<
 *                 if f in ('-ccbin', '--compiler-bindir'):
 *                     _nvcc[i+1] = _cc[0]
*/
      __pyx_t_6 = 0;
      __pyx_t_2 = __pyx_v_4cupy_3fft_9_callback__nvcc; __Pyx_INCREF(__pyx_t_2);
      __pyx_t_7 = 0;
      for (;;) {
        {
          Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2);
          #if !CYTHON_ASSUME_SAFE_SIZE
          if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 138, __pyx_L1_error)
          #endif
          if (__pyx_t_7 >= __pyx_temp) break;
        }
        __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_7, __Pyx_ReferenceSharing_OwnStrongReference);
        ++__pyx_t_7;
        if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 138, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
        if (!(likely(PyUnicode_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_4))) __PYX_ERR(0, 138, __pyx_L1_error)
        __Pyx_XDECREF_SET(__pyx_v_f, ((PyObject*)__pyx_t_4));
        __pyx_t_4 = 0;
        __pyx_v_i = __pyx_t_6;
        __pyx_t_6 = (__pyx_t_6 + 1);

        /* "cupy/fft/_callback.pyx":139
 *             # the one reported by sysconfig
 *             for i, f in enumerate(_nvcc):
 *                 if f in ('-ccbin', '--compiler-bindir'):             # <<<<<<<<<<<<<<
 *                     _nvcc[i+1] = _cc[0]
 *                     break
*/
        __Pyx_INCREF(__pyx_v_f);
        __pyx_t_8 = __pyx_v_f;
        __pyx_t_9 = (__Pyx_PyUnicode_Equals(__pyx_t_8, __pyx_mstate_global->__pyx_kp_u_ccbin, Py_EQ)); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 139, __pyx_L1_error)
        if (!__pyx_t_9) {
        } else {
          __pyx_t_3 = __pyx_t_9;
          goto __pyx_L8_bool_binop_done;
        }
        __pyx_t_9 = (__Pyx_PyUnicode_Equals(__pyx_t_8, __pyx_mstate_global->__pyx_kp_u_compiler_bindir, Py_EQ)); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 139, __pyx_L1_error)
        __pyx_t_3 = __pyx_t_9;
        __pyx_L8_bool_binop_done:;
        __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
        __pyx_t_9 = __pyx_t_3;
        if (__pyx_t_9) {

          /* "cupy/fft/_callback.pyx":140
 *             for i, f in enumerate(_nvcc):
 *                 if f in ('-ccbin', '--compiler-bindir'):
 *                     _nvcc[i+1] = _cc[0]             # <<<<<<<<<<<<<<
 *                     break
 *             else:
*/
          if (unlikely(__pyx_v_4cupy_3fft_9_callback__cc == Py_None)) {
            PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
            __PYX_ERR(0, 140, __pyx_L1_error)
          }
          __pyx_t_4 = __Pyx_GetItemInt_List(__pyx_v_4cupy_3fft_9_callback__cc, 0, long, 1, __Pyx_PyLong_From_long, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 140, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_4);
          if (unlikely(__pyx_v_4cupy_3fft_9_callback__nvcc == Py_None)) {
            PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
            __PYX_ERR(0, 140, __pyx_L1_error)
          }
          __pyx_t_10 = (__pyx_v_i + 1);
          if (unlikely((__Pyx_SetItemInt(__pyx_v_4cupy_3fft_9_callback__nvcc, __pyx_t_10, __pyx_t_4, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference) < 0))) __PYX_ERR(0, 140, __pyx_L1_error)
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;

          /* "cupy/fft/_callback.pyx":141
 *                 if f in ('-ccbin', '--compiler-bindir'):
 *                     _nvcc[i+1] = _cc[0]
 *                     break             # <<<<<<<<<<<<<<
 *             else:
 *                 _nvcc += ['-ccbin', _cc[0]]
*/
          goto __pyx_L6_break;

          /* "cupy/fft/_callback.pyx":139
 *             # the one reported by sysconfig
 *             for i, f in enumerate(_nvcc):
 *                 if f in ('-ccbin', '--compiler-bindir'):             # <<<<<<<<<<<<<<
 *                     _nvcc[i+1] = _cc[0]
 *                     break
*/
        }

        /* "cupy/fft/_callback.pyx":138
 *             # if the host compiler is set in NVCC, we force it to align with
 *             # the one reported by sysconfig
 *             for i, f in enumerate(_nvcc):             # <<<<<<<<<<<<<<
 *                 if f in ('-ccbin', '--compiler-bindir'):
 *                     _nvcc[i+1] = _cc[0]
*/
      }
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      goto __pyx_L10_for_else;
      __pyx_L6_break:;
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      goto __pyx_L11_for_end;
      /*else*/ {
        __pyx_L10_for_else:;

        /* "cupy/fft/_callback.pyx":143
 *                     break
 *             else:
 *                 _nvcc += ['-ccbin', _cc[0]]             # <<<<<<<<<<<<<<
 *         else:
 *             _nvcc = None
*/
        if (unlikely(__pyx_v_4cupy_3fft_9_callback__cc == Py_None)) {
          PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
          __PYX_ERR(0, 143, __pyx_L1_error)
        }
        __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_4cupy_3fft_9_callback__cc, 0, long, 1, __Pyx_PyLong_From_long, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 143, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
        __pyx_t_4 = PyList_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
        __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_ccbin);
        __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_ccbin);
        if (__Pyx_PyList_SET_ITEM(__pyx_t_4, 0, __pyx_mstate_global->__pyx_kp_u_ccbin) != (0)) __PYX_ERR(0, 143, __pyx_L1_error);
        __Pyx_GIVEREF(__pyx_t_2);
        if (__Pyx_PyList_SET_ITEM(__pyx_t_4, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 143, __pyx_L1_error);
        __pyx_t_2 = 0;
        __pyx_t_2 = PyNumber_InPlaceAdd(__pyx_v_4cupy_3fft_9_callback__nvcc, __pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 143, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvcc);
        __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvcc, ((PyObject*)__pyx_t_2));
        __Pyx_GIVEREF(__pyx_t_2);
        __pyx_t_2 = 0;
      }
      __pyx_L11_for_end:;

      /* "cupy/fft/_callback.pyx":134
 *     if _nvcc == []:
 *         nvcc = get_nvcc_path()
 *         if nvcc is not None:             # <<<<<<<<<<<<<<
 *             _nvcc = nvcc.split(' ')
 *             # if the host compiler is set in NVCC, we force it to align with
*/
      goto __pyx_L4;
    }

    /* "cupy/fft/_callback.pyx":145
 *                 _nvcc += ['-ccbin', _cc[0]]
 *         else:
 *             _nvcc = None             # <<<<<<<<<<<<<<
 * 
 * 
*/
    /*else*/ {
      __Pyx_INCREF(Py_None);
      __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvcc);
      __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvcc, ((PyObject*)Py_None));
      __Pyx_GIVEREF(Py_None);
    }
    __pyx_L4:;

    /* "cupy/fft/_callback.pyx":132
 * 
 *     global _nvcc
 *     if _nvcc == []:             # <<<<<<<<<<<<<<
 *         nvcc = get_nvcc_path()
 *         if nvcc is not None:
*/
  }

  /* "cupy/fft/_callback.pyx":125
 * 
 * 
 * cdef inline void _set_nvcc_path() except*:             # <<<<<<<<<<<<<<
 *     # get_nvcc_path() could be a long string like "ccache nvcc ..." from NVCC
 *     cdef str nvcc = None
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_AddTraceback("cupy.fft._callback._set_nvcc_path", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_nvcc);
  __Pyx_XDECREF(__pyx_v_f);
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":148
 * 
 * 
 * cdef inline void _sanity_checks(             # <<<<<<<<<<<<<<
 *         str cb_load, str cb_store,
 *         MemoryPointer cb_load_data, MemoryPointer cb_store_data) except*:
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__sanity_checks(PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data) {
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  size_t __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_t_6;
  int __pyx_t_7;
  int __pyx_t_8;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_sanity_checks", 0);

  /* "cupy/fft/_callback.pyx":151
 *         str cb_load, str cb_store,
 *         MemoryPointer cb_load_data, MemoryPointer cb_store_data) except*:
 *     if runtime._is_hip_environment:             # <<<<<<<<<<<<<<
 *         raise RuntimeError('hipFFT does not support callbacks')
 *     if not sys.platform.startswith('linux'):
*/
  if (unlikely(__pyx_v_13cupy_backends_4cuda_3api_7runtime__is_hip_environment)) {

    /* "cupy/fft/_callback.pyx":152
 *         MemoryPointer cb_load_data, MemoryPointer cb_store_data) except*:
 *     if runtime._is_hip_environment:
 *         raise RuntimeError('hipFFT does not support callbacks')             # <<<<<<<<<<<<<<
 *     if not sys.platform.startswith('linux'):
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')
*/
    __pyx_t_2 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_kp_u_hipFFT_does_not_support_callback};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 152, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 152, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":151
 *         str cb_load, str cb_store,
 *         MemoryPointer cb_load_data, MemoryPointer cb_store_data) except*:
 *     if runtime._is_hip_environment:             # <<<<<<<<<<<<<<
 *         raise RuntimeError('hipFFT does not support callbacks')
 *     if not sys.platform.startswith('linux'):
*/
  }

  /* "cupy/fft/_callback.pyx":153
 *     if runtime._is_hip_environment:
 *         raise RuntimeError('hipFFT does not support callbacks')
 *     if not sys.platform.startswith('linux'):             # <<<<<<<<<<<<<<
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')
 *     if not (sys.maxsize > 2**32):
*/
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_sys); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 153, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_platform); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 153, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  __pyx_t_2 = __pyx_t_5;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_3 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_linux};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_startswith, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 153, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 153, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_7 = (!__pyx_t_6);
  if (unlikely(__pyx_t_7)) {

    /* "cupy/fft/_callback.pyx":154
 *         raise RuntimeError('hipFFT does not support callbacks')
 *     if not sys.platform.startswith('linux'):
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')             # <<<<<<<<<<<<<<
 *     if not (sys.maxsize > 2**32):
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_cuFFT_callbacks_are_only_availab};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 154, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 154, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":153
 *     if runtime._is_hip_environment:
 *         raise RuntimeError('hipFFT does not support callbacks')
 *     if not sys.platform.startswith('linux'):             # <<<<<<<<<<<<<<
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')
 *     if not (sys.maxsize > 2**32):
*/
  }

  /* "cupy/fft/_callback.pyx":155
 *     if not sys.platform.startswith('linux'):
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')
 *     if not (sys.maxsize > 2**32):             # <<<<<<<<<<<<<<
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')
 *     if not cb_load and not cb_store:
*/
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_sys); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 155, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_maxsize); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 155, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyObject_RichCompare(__pyx_t_5, __pyx_mstate_global->__pyx_int_4294967296, Py_GT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 155, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
  __pyx_t_7 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_7 < 0))) __PYX_ERR(0, 155, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_6 = (!__pyx_t_7);
  if (unlikely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":156
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')
 *     if not (sys.maxsize > 2**32):
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')             # <<<<<<<<<<<<<<
 *     if not cb_load and not cb_store:
 *         raise ValueError('need to specify either cb_load or cb_store, '
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_cuFFT_callbacks_require_64_bit_O};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 156, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 156, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":155
 *     if not sys.platform.startswith('linux'):
 *         raise RuntimeError('cuFFT callbacks are only available on Linux')
 *     if not (sys.maxsize > 2**32):             # <<<<<<<<<<<<<<
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')
 *     if not cb_load and not cb_store:
*/
  }

  /* "cupy/fft/_callback.pyx":157
 *     if not (sys.maxsize > 2**32):
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')
 *     if not cb_load and not cb_store:             # <<<<<<<<<<<<<<
 *         raise ValueError('need to specify either cb_load or cb_store, '
 *                          'or both')
*/
  if (__pyx_v_cb_load == Py_None) __pyx_t_7 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_load);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 157, __pyx_L1_error)
    __pyx_t_7 = (__pyx_temp != 0);
  }

  __pyx_t_8 = (!__pyx_t_7);
  if (__pyx_t_8) {
  } else {
    __pyx_t_6 = __pyx_t_8;
    goto __pyx_L7_bool_binop_done;
  }
  if (__pyx_v_cb_store == Py_None) __pyx_t_8 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_store);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 157, __pyx_L1_error)
    __pyx_t_8 = (__pyx_temp != 0);
  }

  __pyx_t_7 = (!__pyx_t_8);
  __pyx_t_6 = __pyx_t_7;
  __pyx_L7_bool_binop_done:;
  if (unlikely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":158
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')
 *     if not cb_load and not cb_store:
 *         raise ValueError('need to specify either cb_load or cb_store, '             # <<<<<<<<<<<<<<
 *                          'or both')
 *     if cb_load and 'd_loadCallbackPtr' not in cb_load:
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_need_to_specify_either_cb_load_o};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 158, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 158, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":157
 *     if not (sys.maxsize > 2**32):
 *         raise RuntimeError('cuFFT callbacks require 64 bit OS')
 *     if not cb_load and not cb_store:             # <<<<<<<<<<<<<<
 *         raise ValueError('need to specify either cb_load or cb_store, '
 *                          'or both')
*/
  }

  /* "cupy/fft/_callback.pyx":160
 *         raise ValueError('need to specify either cb_load or cb_store, '
 *                          'or both')
 *     if cb_load and 'd_loadCallbackPtr' not in cb_load:             # <<<<<<<<<<<<<<
 *         raise ValueError('need to specify d_loadCallbackPtr in cb_load')
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:
*/
  if (__pyx_v_cb_load == Py_None) __pyx_t_7 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_load);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 160, __pyx_L1_error)
    __pyx_t_7 = (__pyx_temp != 0);
  }

  if (__pyx_t_7) {
  } else {
    __pyx_t_6 = __pyx_t_7;
    goto __pyx_L10_bool_binop_done;
  }
  if (unlikely(__pyx_v_cb_load == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
    __PYX_ERR(0, 160, __pyx_L1_error)
  }
  __pyx_t_7 = (__Pyx_PyUnicode_ContainsTF(__pyx_mstate_global->__pyx_n_u_d_loadCallbackPtr, __pyx_v_cb_load, Py_NE)); if (unlikely((__pyx_t_7 < 0))) __PYX_ERR(0, 160, __pyx_L1_error)
  __pyx_t_6 = __pyx_t_7;
  __pyx_L10_bool_binop_done:;
  if (unlikely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":161
 *                          'or both')
 *     if cb_load and 'd_loadCallbackPtr' not in cb_load:
 *         raise ValueError('need to specify d_loadCallbackPtr in cb_load')             # <<<<<<<<<<<<<<
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_need_to_specify_d_loadCallbackPt};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 161, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 161, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":160
 *         raise ValueError('need to specify either cb_load or cb_store, '
 *                          'or both')
 *     if cb_load and 'd_loadCallbackPtr' not in cb_load:             # <<<<<<<<<<<<<<
 *         raise ValueError('need to specify d_loadCallbackPtr in cb_load')
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:
*/
  }

  /* "cupy/fft/_callback.pyx":162
 *     if cb_load and 'd_loadCallbackPtr' not in cb_load:
 *         raise ValueError('need to specify d_loadCallbackPtr in cb_load')
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:             # <<<<<<<<<<<<<<
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')
 *     if _cc is None:
*/
  if (__pyx_v_cb_store == Py_None) __pyx_t_7 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_store);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 162, __pyx_L1_error)
    __pyx_t_7 = (__pyx_temp != 0);
  }

  if (__pyx_t_7) {
  } else {
    __pyx_t_6 = __pyx_t_7;
    goto __pyx_L13_bool_binop_done;
  }
  if (unlikely(__pyx_v_cb_store == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
    __PYX_ERR(0, 162, __pyx_L1_error)
  }
  __pyx_t_7 = (__Pyx_PyUnicode_ContainsTF(__pyx_mstate_global->__pyx_n_u_d_storeCallbackPtr, __pyx_v_cb_store, Py_NE)); if (unlikely((__pyx_t_7 < 0))) __PYX_ERR(0, 162, __pyx_L1_error)
  __pyx_t_6 = __pyx_t_7;
  __pyx_L13_bool_binop_done:;
  if (unlikely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":163
 *         raise ValueError('need to specify d_loadCallbackPtr in cb_load')
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')             # <<<<<<<<<<<<<<
 *     if _cc is None:
 *         raise RuntimeError('a C++ compiler is required but not found, '
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_need_to_specify_d_storeCallbackP};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 163, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 163, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":162
 *     if cb_load and 'd_loadCallbackPtr' not in cb_load:
 *         raise ValueError('need to specify d_loadCallbackPtr in cb_load')
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:             # <<<<<<<<<<<<<<
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')
 *     if _cc is None:
*/
  }

  /* "cupy/fft/_callback.pyx":164
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')
 *     if _cc is None:             # <<<<<<<<<<<<<<
 *         raise RuntimeError('a C++ compiler is required but not found, '
 *                            'please set the environment variable CXX')
*/
  __pyx_t_6 = (__pyx_v_4cupy_3fft_9_callback__cc == ((PyObject*)Py_None));
  if (unlikely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":165
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')
 *     if _cc is None:
 *         raise RuntimeError('a C++ compiler is required but not found, '             # <<<<<<<<<<<<<<
 *                            'please set the environment variable CXX')
 *     if _nvcc is None:
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_a_C_compiler_is_required_but_not};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 165, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 165, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":164
 *     if cb_store and 'd_storeCallbackPtr' not in cb_store:
 *         raise ValueError('need to specify d_storeCallbackPtr in cb_store')
 *     if _cc is None:             # <<<<<<<<<<<<<<
 *         raise RuntimeError('a C++ compiler is required but not found, '
 *                            'please set the environment variable CXX')
*/
  }

  /* "cupy/fft/_callback.pyx":167
 *         raise RuntimeError('a C++ compiler is required but not found, '
 *                            'please set the environment variable CXX')
 *     if _nvcc is None:             # <<<<<<<<<<<<<<
 *         raise RuntimeError('nvcc is required but not found')
 *     if _nvprune is None:
*/
  __pyx_t_6 = (__pyx_v_4cupy_3fft_9_callback__nvcc == ((PyObject*)Py_None));
  if (unlikely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":168
 *                            'please set the environment variable CXX')
 *     if _nvcc is None:
 *         raise RuntimeError('nvcc is required but not found')             # <<<<<<<<<<<<<<
 *     if _nvprune is None:
 *         warnings.warn('nvprune is not found', RuntimeWarning)
*/
    __pyx_t_5 = NULL;
    __pyx_t_3 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_nvcc_is_required_but_not_found};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 168, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 168, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":167
 *         raise RuntimeError('a C++ compiler is required but not found, '
 *                            'please set the environment variable CXX')
 *     if _nvcc is None:             # <<<<<<<<<<<<<<
 *         raise RuntimeError('nvcc is required but not found')
 *     if _nvprune is None:
*/
  }

  /* "cupy/fft/_callback.pyx":169
 *     if _nvcc is None:
 *         raise RuntimeError('nvcc is required but not found')
 *     if _nvprune is None:             # <<<<<<<<<<<<<<
 *         warnings.warn('nvprune is not found', RuntimeWarning)
 *     if cb_load_data is not None:
*/
  __pyx_t_6 = (__pyx_v_4cupy_3fft_9_callback__nvprune == ((PyObject*)Py_None));
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":170
 *         raise RuntimeError('nvcc is required but not found')
 *     if _nvprune is None:
 *         warnings.warn('nvprune is not found', RuntimeWarning)             # <<<<<<<<<<<<<<
 *     if cb_load_data is not None:
 *         if not cb_load:
*/
    __pyx_t_5 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_warnings); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 170, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_warn); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 170, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __pyx_t_3 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_4))) {
      __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4);
      assert(__pyx_t_5);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
      __Pyx_INCREF(__pyx_t_5);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
      __pyx_t_3 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_nvprune_is_not_found, ((PyObject *)(((PyTypeObject*)PyExc_RuntimeWarning)))};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_3, (3-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 170, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":169
 *     if _nvcc is None:
 *         raise RuntimeError('nvcc is required but not found')
 *     if _nvprune is None:             # <<<<<<<<<<<<<<
 *         warnings.warn('nvprune is not found', RuntimeWarning)
 *     if cb_load_data is not None:
*/
  }

  /* "cupy/fft/_callback.pyx":171
 *     if _nvprune is None:
 *         warnings.warn('nvprune is not found', RuntimeWarning)
 *     if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *         if not cb_load:
 *             raise ValueError('load callback is not given')
*/
  __pyx_t_6 = (((PyObject *)__pyx_v_cb_load_data) != Py_None);
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":172
 *         warnings.warn('nvprune is not found', RuntimeWarning)
 *     if cb_load_data is not None:
 *         if not cb_load:             # <<<<<<<<<<<<<<
 *             raise ValueError('load callback is not given')
 *     if cb_store_data is not None:
*/
    if (__pyx_v_cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 172, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    __pyx_t_7 = (!__pyx_t_6);
    if (unlikely(__pyx_t_7)) {

      /* "cupy/fft/_callback.pyx":173
 *     if cb_load_data is not None:
 *         if not cb_load:
 *             raise ValueError('load callback is not given')             # <<<<<<<<<<<<<<
 *     if cb_store_data is not None:
 *         if not cb_store:
*/
      __pyx_t_4 = NULL;
      __pyx_t_3 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_load_callback_is_not_given};
        __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 173, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_1);
      }
      __Pyx_Raise(__pyx_t_1, 0, 0, 0);
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      __PYX_ERR(0, 173, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":172
 *         warnings.warn('nvprune is not found', RuntimeWarning)
 *     if cb_load_data is not None:
 *         if not cb_load:             # <<<<<<<<<<<<<<
 *             raise ValueError('load callback is not given')
 *     if cb_store_data is not None:
*/
    }

    /* "cupy/fft/_callback.pyx":171
 *     if _nvprune is None:
 *         warnings.warn('nvprune is not found', RuntimeWarning)
 *     if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *         if not cb_load:
 *             raise ValueError('load callback is not given')
*/
  }

  /* "cupy/fft/_callback.pyx":174
 *         if not cb_load:
 *             raise ValueError('load callback is not given')
 *     if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *         if not cb_store:
 *             raise ValueError('store callback is not given')
*/
  __pyx_t_7 = (((PyObject *)__pyx_v_cb_store_data) != Py_None);
  if (__pyx_t_7) {

    /* "cupy/fft/_callback.pyx":175
 *             raise ValueError('load callback is not given')
 *     if cb_store_data is not None:
 *         if not cb_store:             # <<<<<<<<<<<<<<
 *             raise ValueError('store callback is not given')
 * 
*/
    if (__pyx_v_cb_store == Py_None) __pyx_t_7 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 175, __pyx_L1_error)
      __pyx_t_7 = (__pyx_temp != 0);
    }

    __pyx_t_6 = (!__pyx_t_7);
    if (unlikely(__pyx_t_6)) {

      /* "cupy/fft/_callback.pyx":176
 *     if cb_store_data is not None:
 *         if not cb_store:
 *             raise ValueError('store callback is not given')             # <<<<<<<<<<<<<<
 * 
 * 
*/
      __pyx_t_4 = NULL;
      __pyx_t_3 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_store_callback_is_not_given};
        __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 176, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_1);
      }
      __Pyx_Raise(__pyx_t_1, 0, 0, 0);
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      __PYX_ERR(0, 176, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":175
 *             raise ValueError('load callback is not given')
 *     if cb_store_data is not None:
 *         if not cb_store:             # <<<<<<<<<<<<<<
 *             raise ValueError('store callback is not given')
 * 
*/
    }

    /* "cupy/fft/_callback.pyx":174
 *         if not cb_load:
 *             raise ValueError('load callback is not given')
 *     if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *         if not cb_store:
 *             raise ValueError('store callback is not given')
*/
  }

  /* "cupy/fft/_callback.pyx":148
 * 
 * 
 * cdef inline void _sanity_checks(             # <<<<<<<<<<<<<<
 *         str cb_load, str cb_store,
 *         MemoryPointer cb_load_data, MemoryPointer cb_store_data) except*:
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback._sanity_checks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":179
 * 
 * 
 * cdef inline void _cythonize(str tempdir, str mod_name) except*:             # <<<<<<<<<<<<<<
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufft.h'),
 *                     os.path.join(tempdir, 'cupy_cufft.h'))
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__cythonize(PyObject *__pyx_v_tempdir, PyObject *__pyx_v_mod_name) {
  PyObject *__pyx_v_p = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  size_t __pyx_t_8;
  PyObject *__pyx_t_9 = NULL;
  PyObject *__pyx_t_10 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_cythonize", 0);

  /* "cupy/fft/_callback.pyx":180
 * 
 * cdef inline void _cythonize(str tempdir, str mod_name) except*:
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufft.h'),             # <<<<<<<<<<<<<<
 *                     os.path.join(tempdir, 'cupy_cufft.h'))
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pxd'),
*/
  __pyx_t_2 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_shutil); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 180, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_copyfile); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 180, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 180, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 180, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_5 = __pyx_t_7;
  __Pyx_INCREF(__pyx_t_5);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_4cupy_3fft_9_callback__source_dir, __pyx_mstate_global->__pyx_kp_u_cupy_cufft_h};
    __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 180, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
  }

  /* "cupy/fft/_callback.pyx":181
 * cdef inline void _cythonize(str tempdir, str mod_name) except*:
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufft.h'),
 *                     os.path.join(tempdir, 'cupy_cufft.h'))             # <<<<<<<<<<<<<<
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pxd'),
 *                     os.path.join(tempdir, mod_name + '.pxd'))
*/
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 181, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 181, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_5 = __pyx_t_9;
  __Pyx_INCREF(__pyx_t_5);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_tempdir, __pyx_mstate_global->__pyx_kp_u_cupy_cufft_h};
    __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 181, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
  }
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_4))) {
    __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_4);
    assert(__pyx_t_2);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
    __Pyx_INCREF(__pyx_t_2);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_t_3, __pyx_t_7};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 180, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":182
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufft.h'),
 *                     os.path.join(tempdir, 'cupy_cufft.h'))
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pxd'),             # <<<<<<<<<<<<<<
 *                     os.path.join(tempdir, mod_name + '.pxd'))
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_shutil); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 182, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_copyfile); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 182, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
  __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 182, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 182, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
  __pyx_t_2 = __pyx_t_5;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_4cupy_3fft_9_callback__source_dir, __pyx_mstate_global->__pyx_kp_u_cufft_pxd};
    __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 182, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
  }

  /* "cupy/fft/_callback.pyx":183
 *                     os.path.join(tempdir, 'cupy_cufft.h'))
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pxd'),
 *                     os.path.join(tempdir, mod_name + '.pxd'))             # <<<<<<<<<<<<<<
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),
 *                     os.path.join(tempdir, mod_name + '.pyx'))
*/
  __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 183, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 183, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
  __pyx_t_2 = __pyx_t_6;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_9 = __Pyx_PyUnicode_ConcatSafe(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_pxd); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 183, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_tempdir, __pyx_t_9};
    __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 183, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
  }
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_3))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_3, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_t_7, __pyx_t_5};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 182, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":184
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pxd'),
 *                     os.path.join(tempdir, mod_name + '.pxd'))
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),             # <<<<<<<<<<<<<<
 *                     os.path.join(tempdir, mod_name + '.pyx'))
 *     p = subprocess.run(['cython', '-3', '--cplus',
*/
  __pyx_t_3 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_shutil); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 184, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_copyfile); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 184, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 184, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 184, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_4 = __pyx_t_9;
  __Pyx_INCREF(__pyx_t_4);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_4cupy_3fft_9_callback__source_dir, __pyx_mstate_global->__pyx_kp_u_cufft_pyx};
    __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 184, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
  }

  /* "cupy/fft/_callback.pyx":185
 *                     os.path.join(tempdir, mod_name + '.pxd'))
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),
 *                     os.path.join(tempdir, mod_name + '.pyx'))             # <<<<<<<<<<<<<<
 *     p = subprocess.run(['cython', '-3', '--cplus',
 *                         '-E', 'CUPY_HIP_VERSION=0',
*/
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 185, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_4 = __pyx_t_2;
  __Pyx_INCREF(__pyx_t_4);
  __pyx_t_6 = __Pyx_PyUnicode_ConcatSafe(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_pyx); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 185, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_tempdir, __pyx_t_6};
    __pyx_t_9 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 185, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
  }
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_7))) {
    __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_7);
    assert(__pyx_t_3);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7);
    __Pyx_INCREF(__pyx_t_3);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_7, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_5, __pyx_t_9};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 184, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":186
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),
 *                     os.path.join(tempdir, mod_name + '.pyx'))
 *     p = subprocess.run(['cython', '-3', '--cplus',             # <<<<<<<<<<<<<<
 *                         '-E', 'CUPY_HIP_VERSION=0',
 *                         '-E', 'CUPY_CUDA_VERSION=' + _build_ver,
*/
  __pyx_t_7 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 186, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_run); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 186, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;

  /* "cupy/fft/_callback.pyx":188
 *     p = subprocess.run(['cython', '-3', '--cplus',
 *                         '-E', 'CUPY_HIP_VERSION=0',
 *                         '-E', 'CUPY_CUDA_VERSION=' + _build_ver,             # <<<<<<<<<<<<<<
 *                         '-E', 'CUPY_CUFFT_STATIC=True',
 *                         os.path.join(tempdir, mod_name + '.pyx'),
*/
  __pyx_t_9 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_CUPY_CUDA_VERSION, __pyx_v_4cupy_3fft_9_callback__build_ver); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 188, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);

  /* "cupy/fft/_callback.pyx":190
 *                         '-E', 'CUPY_CUDA_VERSION=' + _build_ver,
 *                         '-E', 'CUPY_CUFFT_STATIC=True',
 *                         os.path.join(tempdir, mod_name + '.pyx'),             # <<<<<<<<<<<<<<
 *                         '-o', os.path.join(tempdir, mod_name + '.cpp')],
 *                        env=os.environ, cwd=tempdir)
*/
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 190, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 190, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_2 = __pyx_t_4;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_6 = __Pyx_PyUnicode_ConcatSafe(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_pyx); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 190, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_tempdir, __pyx_t_6};
    __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 190, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
  }

  /* "cupy/fft/_callback.pyx":191
 *                         '-E', 'CUPY_CUFFT_STATIC=True',
 *                         os.path.join(tempdir, mod_name + '.pyx'),
 *                         '-o', os.path.join(tempdir, mod_name + '.cpp')],             # <<<<<<<<<<<<<<
 *                        env=os.environ, cwd=tempdir)
 *     p.check_returncode()
*/
  __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 191, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 191, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_10);
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  __pyx_t_6 = __pyx_t_10;
  __Pyx_INCREF(__pyx_t_6);
  __pyx_t_2 = __Pyx_PyUnicode_ConcatSafe(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_cpp); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 191, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_v_tempdir, __pyx_t_2};
    __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
    if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 191, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
  }

  /* "cupy/fft/_callback.pyx":186
 *     shutil.copyfile(os.path.join(_source_dir, 'cufft.pyx'),
 *                     os.path.join(tempdir, mod_name + '.pyx'))
 *     p = subprocess.run(['cython', '-3', '--cplus',             # <<<<<<<<<<<<<<
 *                         '-E', 'CUPY_HIP_VERSION=0',
 *                         '-E', 'CUPY_CUDA_VERSION=' + _build_ver,
*/
  __pyx_t_10 = PyList_New(12); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 186, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_10);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_cython);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_cython);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 0, __pyx_mstate_global->__pyx_n_u_cython) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_3);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_3);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 1, __pyx_mstate_global->__pyx_kp_u_3) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_cplus);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_cplus);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 2, __pyx_mstate_global->__pyx_kp_u_cplus) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_E);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_E);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 3, __pyx_mstate_global->__pyx_kp_u_E) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_CUPY_HIP_VERSION_0);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_CUPY_HIP_VERSION_0);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 4, __pyx_mstate_global->__pyx_kp_u_CUPY_HIP_VERSION_0) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_E);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_E);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 5, __pyx_mstate_global->__pyx_kp_u_E) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_9);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 6, __pyx_t_9) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_E);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_E);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 7, __pyx_mstate_global->__pyx_kp_u_E) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_CUPY_CUFFT_STATIC_True);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_CUPY_CUFFT_STATIC_True);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 8, __pyx_mstate_global->__pyx_kp_u_CUPY_CUFFT_STATIC_True) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_3);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 9, __pyx_t_3) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_o);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_o);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 10, __pyx_mstate_global->__pyx_kp_u_o) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_4);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 11, __pyx_t_4) != (0)) __PYX_ERR(0, 186, __pyx_L1_error);
  __pyx_t_9 = 0;
  __pyx_t_3 = 0;
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":192
 *                         os.path.join(tempdir, mod_name + '.pyx'),
 *                         '-o', os.path.join(tempdir, mod_name + '.cpp')],
 *                        env=os.environ, cwd=tempdir)             # <<<<<<<<<<<<<<
 *     p.check_returncode()
 * 
*/
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 192, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 192, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_5))) {
    __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_5);
    assert(__pyx_t_7);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5);
    __Pyx_INCREF(__pyx_t_7);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_5, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_7, __pyx_t_10};
    __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 186, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_env, __pyx_t_3, __pyx_t_4, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 186, __pyx_L1_error)
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cwd, __pyx_v_tempdir, __pyx_t_4, __pyx_callargs+2, 1) < (0)) __PYX_ERR(0, 186, __pyx_L1_error)
    __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4);
    __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 186, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __pyx_v_p = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":193
 *                         '-o', os.path.join(tempdir, mod_name + '.cpp')],
 *                        env=os.environ, cwd=tempdir)
 *     p.check_returncode()             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __pyx_t_5 = __pyx_v_p;
  __Pyx_INCREF(__pyx_t_5);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_5, NULL};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_check_returncode, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 193, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":179
 * 
 * 
 * cdef inline void _cythonize(str tempdir, str mod_name) except*:             # <<<<<<<<<<<<<<
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufft.h'),
 *                     os.path.join(tempdir, 'cupy_cufft.h'))
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_XDECREF(__pyx_t_10);
  __Pyx_AddTraceback("cupy.fft._callback._cythonize", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_p);
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":196
 * 
 * 
 * cdef inline void _mod_compile(str tempdir, str mod_name, str obj_host) except*:             # <<<<<<<<<<<<<<
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__mod_compile(PyObject *__pyx_v_tempdir, PyObject *__pyx_v_mod_name, PyObject *__pyx_v_obj_host) {
  PyObject *__pyx_v_p = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  size_t __pyx_t_8;
  PyObject *__pyx_t_9 = NULL;
  PyObject *__pyx_t_10 = NULL;
  PyObject *__pyx_t_11 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_mod_compile", 0);

  /* "cupy/fft/_callback.pyx":197
 * 
 * cdef inline void _mod_compile(str tempdir, str mod_name, str obj_host) except*:
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),             # <<<<<<<<<<<<<<
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))
 *     p = subprocess.run(_cc + [
*/
  __pyx_t_2 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_shutil); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 197, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_copyfile); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 197, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 197, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 197, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_5 = __pyx_t_7;
  __Pyx_INCREF(__pyx_t_5);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_4cupy_3fft_9_callback__source_dir, __pyx_mstate_global->__pyx_kp_u_cupy_cufftXt_h};
    __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 197, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
  }

  /* "cupy/fft/_callback.pyx":198
 * cdef inline void _mod_compile(str tempdir, str mod_name, str obj_host) except*:
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))             # <<<<<<<<<<<<<<
 *     p = subprocess.run(_cc + [
 *                        '-I' + _python_include,
*/
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 198, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 198, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_5 = __pyx_t_9;
  __Pyx_INCREF(__pyx_t_5);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_tempdir, __pyx_mstate_global->__pyx_kp_u_cupy_cufftXt_h};
    __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
  }
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_4))) {
    __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_4);
    assert(__pyx_t_2);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
    __Pyx_INCREF(__pyx_t_2);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_t_3, __pyx_t_7};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 197, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":199
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))
 *     p = subprocess.run(_cc + [             # <<<<<<<<<<<<<<
 *                        '-I' + _python_include,
 *                        '-I' + _cuda_include,
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 199, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_run); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 199, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

  /* "cupy/fft/_callback.pyx":200
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))
 *     p = subprocess.run(_cc + [
 *                        '-I' + _python_include,             # <<<<<<<<<<<<<<
 *                        '-I' + _cuda_include,
 *                        '-I' + _cupy_include,
*/
  __pyx_t_7 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_I, __pyx_v_4cupy_3fft_9_callback__python_include); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 200, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);

  /* "cupy/fft/_callback.pyx":201
 *     p = subprocess.run(_cc + [
 *                        '-I' + _python_include,
 *                        '-I' + _cuda_include,             # <<<<<<<<<<<<<<
 *                        '-I' + _cupy_include,
 *                        '-fPIC', '-O2',
*/
  __pyx_t_2 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_I, __pyx_v_4cupy_3fft_9_callback__cuda_include); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 201, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);

  /* "cupy/fft/_callback.pyx":202
 *                        '-I' + _python_include,
 *                        '-I' + _cuda_include,
 *                        '-I' + _cupy_include,             # <<<<<<<<<<<<<<
 *                        '-fPIC', '-O2',
 *                        '-c', os.path.join(tempdir, mod_name + '.cpp'),
*/
  __pyx_t_9 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_I, __pyx_v_4cupy_3fft_9_callback__cupy_include); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 202, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);

  /* "cupy/fft/_callback.pyx":204
 *                        '-I' + _cupy_include,
 *                        '-fPIC', '-O2',
 *                        '-c', os.path.join(tempdir, mod_name + '.cpp'),             # <<<<<<<<<<<<<<
 *                        '-o', obj_host],
 *                        env=os.environ, cwd=tempdir)
*/
  __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 204, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_10);
  __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 204, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_11);
  __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
  __pyx_t_6 = __pyx_t_11;
  __Pyx_INCREF(__pyx_t_6);
  __pyx_t_10 = __Pyx_PyUnicode_ConcatSafe(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_cpp); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 204, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_10);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_v_tempdir, __pyx_t_10};
    __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
    __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
    if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 204, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
  }

  /* "cupy/fft/_callback.pyx":199
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))
 *     p = subprocess.run(_cc + [             # <<<<<<<<<<<<<<
 *                        '-I' + _python_include,
 *                        '-I' + _cuda_include,
*/
  __pyx_t_11 = PyList_New(9); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 199, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_11);
  __Pyx_GIVEREF(__pyx_t_7);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_2);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_9);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 2, __pyx_t_9) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_fPIC);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_fPIC);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 3, __pyx_mstate_global->__pyx_kp_u_fPIC) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_O2);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_O2);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 4, __pyx_mstate_global->__pyx_kp_u_O2) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_c);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_c);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 5, __pyx_mstate_global->__pyx_kp_u_c) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_5);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 6, __pyx_t_5) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_o);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_o);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 7, __pyx_mstate_global->__pyx_kp_u_o) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_obj_host);
  __Pyx_GIVEREF(__pyx_v_obj_host);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_11, 8, __pyx_v_obj_host) != (0)) __PYX_ERR(0, 199, __pyx_L1_error);
  __pyx_t_7 = 0;
  __pyx_t_2 = 0;
  __pyx_t_9 = 0;
  __pyx_t_5 = 0;
  __pyx_t_5 = PyNumber_Add(__pyx_v_4cupy_3fft_9_callback__cc, __pyx_t_11); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 199, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;

  /* "cupy/fft/_callback.pyx":206
 *                        '-c', os.path.join(tempdir, mod_name + '.cpp'),
 *                        '-o', obj_host],
 *                        env=os.environ, cwd=tempdir)             # <<<<<<<<<<<<<<
 *     p.check_returncode()
 * 
*/
  __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 206, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_11);
  __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 206, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_9);
  __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_3))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_3, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, __pyx_t_5};
    __pyx_t_11 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 199, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_11);
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_env, __pyx_t_9, __pyx_t_11, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 199, __pyx_L1_error)
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cwd, __pyx_v_tempdir, __pyx_t_11, __pyx_callargs+2, 1) < (0)) __PYX_ERR(0, 199, __pyx_L1_error)
    __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_11);
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 199, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __pyx_v_p = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":207
 *                        '-o', obj_host],
 *                        env=os.environ, cwd=tempdir)
 *     p.check_returncode()             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __pyx_t_3 = __pyx_v_p;
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_check_returncode, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 207, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":196
 * 
 * 
 * cdef inline void _mod_compile(str tempdir, str mod_name, str obj_host) except*:             # <<<<<<<<<<<<<<
 *     shutil.copyfile(os.path.join(_source_dir, 'cupy_cufftXt.h'),
 *                     os.path.join(tempdir, 'cupy_cufftXt.h'))
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_XDECREF(__pyx_t_10);
  __Pyx_XDECREF(__pyx_t_11);
  __Pyx_AddTraceback("cupy.fft._callback._mod_compile", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_p);
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":224
 * 
 * 
 * cdef inline str _prune(str temp_dir, str cache_dir, str _cufft_ver, str arch):             # <<<<<<<<<<<<<<
 *     cdef str cufft_lib_full, cufft_lib_pruned, cufft_lib_temp, cufft_lib_cached
 * 
*/

static CYTHON_INLINE PyObject *__pyx_f_4cupy_3fft_9_callback__prune(PyObject *__pyx_v_temp_dir, PyObject *__pyx_v_cache_dir, PyObject *__pyx_v__cufft_ver, PyObject *__pyx_v_arch) {
  PyObject *__pyx_v_cufft_lib_full = 0;
  PyObject *__pyx_v_cufft_lib_pruned = 0;
  PyObject *__pyx_v_cufft_lib_temp = 0;
  PyObject *__pyx_v_cufft_lib_cached = 0;
  PyObject *__pyx_v_comm = NULL;
  PyObject *__pyx_v_cc = NULL;
  PyObject *__pyx_v_p = NULL;
  PyObject *__pyx_v_err = NULL;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  size_t __pyx_t_6;
  Py_UCS4 __pyx_t_7;
  PyObject *__pyx_t_8[4];
  PyObject *__pyx_t_9 = NULL;
  int __pyx_t_10;
  Py_ssize_t __pyx_t_11;
  PyObject *(*__pyx_t_12)(PyObject *);
  int __pyx_t_13;
  PyObject *__pyx_t_14 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_prune", 0);

  /* "cupy/fft/_callback.pyx":227
 *     cdef str cufft_lib_full, cufft_lib_pruned, cufft_lib_temp, cufft_lib_cached
 * 
 *     if _nvprune:             # <<<<<<<<<<<<<<
 *         cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
*/
  if (__pyx_v_4cupy_3fft_9_callback__nvprune == Py_None) __pyx_t_1 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_4cupy_3fft_9_callback__nvprune);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 227, __pyx_L1_error)
    __pyx_t_1 = (__pyx_temp != 0);
  }

  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":228
 * 
 *     if _nvprune:
 *         cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')             # <<<<<<<<<<<<<<
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
 *         cufft_lib_temp = os.path.join(temp_dir,
*/
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 228, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 228, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __pyx_t_3 = __pyx_t_5;
    __Pyx_INCREF(__pyx_t_3);
    __pyx_t_6 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_v_4cupy_3fft_9_callback__cuda_path, __pyx_mstate_global->__pyx_kp_u_lib64_libcufft_static_a};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 228, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 228, __pyx_L1_error)
    __pyx_v_cufft_lib_full = ((PyObject*)__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":229
 *     if _nvprune:
 *         cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'             # <<<<<<<<<<<<<<
 *         cufft_lib_temp = os.path.join(temp_dir,
 *                                       'lib' + cufft_lib_pruned + '.a')
*/
    __pyx_t_2 = __Pyx_PyUnicode_Unicode(__pyx_v__cufft_ver); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 229, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_7 = __Pyx_GetItemInt_Unicode(__pyx_v_arch, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(__pyx_t_7 == (Py_UCS4)-1)) __PYX_ERR(0, 229, __pyx_L1_error)
    __pyx_t_5 = __Pyx_PyUnicode_FromOrdinal(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 229, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_8[0] = __pyx_mstate_global->__pyx_n_u_cufft_static;
    __pyx_t_8[1] = __pyx_t_2;
    __pyx_t_8[2] = __pyx_mstate_global->__pyx_n_u_sm;
    __pyx_t_8[3] = __pyx_t_5;
    __pyx_t_3 = __Pyx_PyUnicode_Join(__pyx_t_8, 4, 13 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 3 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5), 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_5));
    if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 229, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __pyx_v_cufft_lib_pruned = ((PyObject*)__pyx_t_3);
    __pyx_t_3 = 0;

    /* "cupy/fft/_callback.pyx":230
 *         cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
 *         cufft_lib_temp = os.path.join(temp_dir,             # <<<<<<<<<<<<<<
 *                                       'lib' + cufft_lib_pruned + '.a')
 *         cufft_lib_cached = os.path.join(cache_dir,
*/
    __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 230, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 230, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __pyx_t_5 = __pyx_t_4;
    __Pyx_INCREF(__pyx_t_5);

    /* "cupy/fft/_callback.pyx":231
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
 *         cufft_lib_temp = os.path.join(temp_dir,
 *                                       'lib' + cufft_lib_pruned + '.a')             # <<<<<<<<<<<<<<
 *         cufft_lib_cached = os.path.join(cache_dir,
 *                                         'lib' + cufft_lib_pruned + '.a')
*/
    __pyx_t_2 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_n_u_lib, __pyx_v_cufft_lib_pruned); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 231, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_9 = __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlace(__pyx_t_2, __pyx_mstate_global->__pyx_kp_u_a); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 231, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __pyx_t_6 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_temp_dir, __pyx_t_9};
      __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 230, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
    }

    /* "cupy/fft/_callback.pyx":230
 *         cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
 *         cufft_lib_temp = os.path.join(temp_dir,             # <<<<<<<<<<<<<<
 *                                       'lib' + cufft_lib_pruned + '.a')
 *         cufft_lib_cached = os.path.join(cache_dir,
*/
    if (!(likely(PyUnicode_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_3))) __PYX_ERR(0, 230, __pyx_L1_error)
    __pyx_v_cufft_lib_temp = ((PyObject*)__pyx_t_3);
    __pyx_t_3 = 0;

    /* "cupy/fft/_callback.pyx":232
 *         cufft_lib_temp = os.path.join(temp_dir,
 *                                       'lib' + cufft_lib_pruned + '.a')
 *         cufft_lib_cached = os.path.join(cache_dir,             # <<<<<<<<<<<<<<
 *                                         'lib' + cufft_lib_pruned + '.a')
 *         if not os.path.isfile(cufft_lib_cached):
*/
    __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 232, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
    __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 232, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    __pyx_t_4 = __pyx_t_5;
    __Pyx_INCREF(__pyx_t_4);

    /* "cupy/fft/_callback.pyx":233
 *                                       'lib' + cufft_lib_pruned + '.a')
 *         cufft_lib_cached = os.path.join(cache_dir,
 *                                         'lib' + cufft_lib_pruned + '.a')             # <<<<<<<<<<<<<<
 *         if not os.path.isfile(cufft_lib_cached):
 *             comm = [_nvprune]
*/
    __pyx_t_9 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_n_u_lib, __pyx_v_cufft_lib_pruned); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 233, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
    __pyx_t_2 = __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlace(__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_a); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 233, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    __pyx_t_6 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_cache_dir, __pyx_t_2};
      __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 232, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
    }

    /* "cupy/fft/_callback.pyx":232
 *         cufft_lib_temp = os.path.join(temp_dir,
 *                                       'lib' + cufft_lib_pruned + '.a')
 *         cufft_lib_cached = os.path.join(cache_dir,             # <<<<<<<<<<<<<<
 *                                         'lib' + cufft_lib_pruned + '.a')
 *         if not os.path.isfile(cufft_lib_cached):
*/
    if (!(likely(PyUnicode_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_3))) __PYX_ERR(0, 232, __pyx_L1_error)
    __pyx_v_cufft_lib_cached = ((PyObject*)__pyx_t_3);
    __pyx_t_3 = 0;

    /* "cupy/fft/_callback.pyx":234
 *         cufft_lib_cached = os.path.join(cache_dir,
 *                                         'lib' + cufft_lib_pruned + '.a')
 *         if not os.path.isfile(cufft_lib_cached):             # <<<<<<<<<<<<<<
 *             comm = [_nvprune]
 *             for cc in _cc_major_map[arch[:-1]]:
*/
    __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 234, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 234, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __pyx_t_5 = __pyx_t_4;
    __Pyx_INCREF(__pyx_t_5);
    __pyx_t_6 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_v_cufft_lib_cached};
      __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_isfile, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 234, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
    }
    __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 234, __pyx_L1_error)
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    __pyx_t_10 = (!__pyx_t_1);
    if (__pyx_t_10) {

      /* "cupy/fft/_callback.pyx":235
 *                                         'lib' + cufft_lib_pruned + '.a')
 *         if not os.path.isfile(cufft_lib_cached):
 *             comm = [_nvprune]             # <<<<<<<<<<<<<<
 *             for cc in _cc_major_map[arch[:-1]]:
 *                 comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')
*/
      __pyx_t_3 = PyList_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 235, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
      __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
      __Pyx_GIVEREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
      if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 0, __pyx_v_4cupy_3fft_9_callback__nvprune) != (0)) __PYX_ERR(0, 235, __pyx_L1_error);
      __pyx_v_comm = ((PyObject*)__pyx_t_3);
      __pyx_t_3 = 0;

      /* "cupy/fft/_callback.pyx":236
 *         if not os.path.isfile(cufft_lib_cached):
 *             comm = [_nvprune]
 *             for cc in _cc_major_map[arch[:-1]]:             # <<<<<<<<<<<<<<
 *                 comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')
 *             comm += [cufft_lib_full, '-o', cufft_lib_temp]
*/
      if (unlikely(__pyx_v_4cupy_3fft_9_callback__cc_major_map == Py_None)) {
        PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
        __PYX_ERR(0, 236, __pyx_L1_error)
      }
      if (unlikely(__pyx_v_arch == Py_None)) {
        PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
        __PYX_ERR(0, 236, __pyx_L1_error)
      }
      __pyx_t_3 = __Pyx_PyUnicode_Substring(__pyx_v_arch, 0, -1L); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 236, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
      __pyx_t_4 = __Pyx_PyDict_GetItem(__pyx_v_4cupy_3fft_9_callback__cc_major_map, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 236, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      if (likely(PyList_CheckExact(__pyx_t_4)) || PyTuple_CheckExact(__pyx_t_4)) {
        __pyx_t_3 = __pyx_t_4; __Pyx_INCREF(__pyx_t_3);
        __pyx_t_11 = 0;
        __pyx_t_12 = NULL;
      } else {
        __pyx_t_11 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 236, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_3);
        __pyx_t_12 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 236, __pyx_L1_error)
      }
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      for (;;) {
        if (likely(!__pyx_t_12)) {
          if (likely(PyList_CheckExact(__pyx_t_3))) {
            {
              Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3);
              #if !CYTHON_ASSUME_SAFE_SIZE
              if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 236, __pyx_L1_error)
              #endif
              if (__pyx_t_11 >= __pyx_temp) break;
            }
            __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_3, __pyx_t_11, __Pyx_ReferenceSharing_OwnStrongReference);
            ++__pyx_t_11;
          } else {
            {
              Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_3);
              #if !CYTHON_ASSUME_SAFE_SIZE
              if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 236, __pyx_L1_error)
              #endif
              if (__pyx_t_11 >= __pyx_temp) break;
            }
            #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
            __pyx_t_4 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_11));
            #else
            __pyx_t_4 = __Pyx_PySequence_ITEM(__pyx_t_3, __pyx_t_11);
            #endif
            ++__pyx_t_11;
          }
          if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 236, __pyx_L1_error)
        } else {
          __pyx_t_4 = __pyx_t_12(__pyx_t_3);
          if (unlikely(!__pyx_t_4)) {
            PyObject* exc_type = PyErr_Occurred();
            if (exc_type) {
              if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 236, __pyx_L1_error)
              PyErr_Clear();
            }
            break;
          }
        }
        __Pyx_GOTREF(__pyx_t_4);
        __Pyx_XDECREF_SET(__pyx_v_cc, __pyx_t_4);
        __pyx_t_4 = 0;

        /* "cupy/fft/_callback.pyx":237
 *             comm = [_nvprune]
 *             for cc in _cc_major_map[arch[:-1]]:
 *                 comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')             # <<<<<<<<<<<<<<
 *             comm += [cufft_lib_full, '-o', cufft_lib_temp]
 *             p = subprocess.run(comm,
*/
        __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_cc, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 237, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
        __pyx_t_5 = __Pyx_PyObject_FormatSimple(__pyx_v_cc, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 237, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_5);
        __pyx_t_8[0] = __pyx_mstate_global->__pyx_kp_u_generate_code_arch_compute;
        __pyx_t_8[1] = __pyx_t_4;
        __pyx_t_8[2] = __pyx_mstate_global->__pyx_kp_u_code_sm;
        __pyx_t_8[3] = __pyx_t_5;
        __pyx_t_2 = __Pyx_PyUnicode_Join(__pyx_t_8, 4, 29 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4) + 9 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5), 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_5));
        if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 237, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
        __pyx_t_13 = __Pyx_PyList_Append(__pyx_v_comm, __pyx_t_2); if (unlikely(__pyx_t_13 == ((int)-1))) __PYX_ERR(0, 237, __pyx_L1_error)
        __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

        /* "cupy/fft/_callback.pyx":236
 *         if not os.path.isfile(cufft_lib_cached):
 *             comm = [_nvprune]
 *             for cc in _cc_major_map[arch[:-1]]:             # <<<<<<<<<<<<<<
 *                 comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')
 *             comm += [cufft_lib_full, '-o', cufft_lib_temp]
*/
      }
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;

      /* "cupy/fft/_callback.pyx":238
 *             for cc in _cc_major_map[arch[:-1]]:
 *                 comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')
 *             comm += [cufft_lib_full, '-o', cufft_lib_temp]             # <<<<<<<<<<<<<<
 *             p = subprocess.run(comm,
 *                                env=os.environ, cwd=temp_dir,
*/
      __pyx_t_3 = PyList_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 238, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
      __Pyx_INCREF(__pyx_v_cufft_lib_full);
      __Pyx_GIVEREF(__pyx_v_cufft_lib_full);
      if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 0, __pyx_v_cufft_lib_full) != (0)) __PYX_ERR(0, 238, __pyx_L1_error);
      __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_o);
      __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_o);
      if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_kp_u_o) != (0)) __PYX_ERR(0, 238, __pyx_L1_error);
      __Pyx_INCREF(__pyx_v_cufft_lib_temp);
      __Pyx_GIVEREF(__pyx_v_cufft_lib_temp);
      if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 2, __pyx_v_cufft_lib_temp) != (0)) __PYX_ERR(0, 238, __pyx_L1_error);
      __pyx_t_2 = PyNumber_InPlaceAdd(__pyx_v_comm, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 238, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF_SET(__pyx_v_comm, ((PyObject*)__pyx_t_2));
      __pyx_t_2 = 0;

      /* "cupy/fft/_callback.pyx":239
 *                 comm.append(f'--generate-code=arch=compute_{cc},code=sm_{cc}')
 *             comm += [cufft_lib_full, '-o', cufft_lib_temp]
 *             p = subprocess.run(comm,             # <<<<<<<<<<<<<<
 *                                env=os.environ, cwd=temp_dir,
 *                                stderr=subprocess.PIPE)
*/
      __pyx_t_3 = NULL;
      __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 239, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_5);
      __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_run); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 239, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;

      /* "cupy/fft/_callback.pyx":240
 *             comm += [cufft_lib_full, '-o', cufft_lib_temp]
 *             p = subprocess.run(comm,
 *                                env=os.environ, cwd=temp_dir,             # <<<<<<<<<<<<<<
 *                                stderr=subprocess.PIPE)
 *             p.check_returncode()
*/
      __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 240, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_5);
      __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 240, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_9);
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;

      /* "cupy/fft/_callback.pyx":241
 *             p = subprocess.run(comm,
 *                                env=os.environ, cwd=temp_dir,
 *                                stderr=subprocess.PIPE)             # <<<<<<<<<<<<<<
 *             p.check_returncode()
 *             if p.stderr:
*/
      __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 241, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_5);
      __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_PIPE); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 241, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_14);
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
      __pyx_t_6 = 1;
      #if CYTHON_UNPACK_METHODS
      if (unlikely(PyMethod_Check(__pyx_t_4))) {
        __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
        assert(__pyx_t_3);
        PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
        __Pyx_INCREF(__pyx_t_3);
        __Pyx_INCREF(__pyx__function);
        __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
        __pyx_t_6 = 0;
      }
      #endif
      {
        PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_3, __pyx_v_comm};
        __pyx_t_5 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 239, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_5);
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_env, __pyx_t_9, __pyx_t_5, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 239, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cwd, __pyx_v_temp_dir, __pyx_t_5, __pyx_callargs+2, 1) < (0)) __PYX_ERR(0, 239, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_stderr, __pyx_t_14, __pyx_t_5, __pyx_callargs+2, 2) < (0)) __PYX_ERR(0, 239, __pyx_L1_error)
        __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_5);
        __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
        __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
        __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
        __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 239, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
      }
      __pyx_v_p = __pyx_t_2;
      __pyx_t_2 = 0;

      /* "cupy/fft/_callback.pyx":242
 *                                env=os.environ, cwd=temp_dir,
 *                                stderr=subprocess.PIPE)
 *             p.check_returncode()             # <<<<<<<<<<<<<<
 *             if p.stderr:
 *                 # maybe this is a new arch that we haven't seen or unsupported
*/
      __pyx_t_4 = __pyx_v_p;
      __Pyx_INCREF(__pyx_t_4);
      __pyx_t_6 = 0;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL};
        __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_check_returncode, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 242, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
      }
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

      /* "cupy/fft/_callback.pyx":243
 *                                stderr=subprocess.PIPE)
 *             p.check_returncode()
 *             if p.stderr:             # <<<<<<<<<<<<<<
 *                 # maybe this is a new arch that we haven't seen or unsupported
 *                 # by the current CUDA ver?
*/
      __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_p, __pyx_mstate_global->__pyx_n_u_stderr); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 243, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
      __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_10 < 0))) __PYX_ERR(0, 243, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      if (unlikely(__pyx_t_10)) {

        /* "cupy/fft/_callback.pyx":246
 *                 # maybe this is a new arch that we haven't seen or unsupported
 *                 # by the current CUDA ver?
 *                 err = f'nvprune failed with sm_{arch}:\n{p.stderr}'             # <<<<<<<<<<<<<<
 *                 raise RuntimeError(err)
 *             else:
*/
        __pyx_t_2 = __Pyx_PyUnicode_Unicode(__pyx_v_arch); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 246, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
        __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_p, __pyx_mstate_global->__pyx_n_u_stderr); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 246, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
        __pyx_t_5 = __Pyx_PyObject_FormatSimple(__pyx_t_4, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 246, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_5);
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        __pyx_t_8[0] = __pyx_mstate_global->__pyx_kp_u_nvprune_failed_with_sm;
        __pyx_t_8[1] = __pyx_t_2;
        __pyx_t_8[2] = __pyx_mstate_global->__pyx_kp_u__5;
        __pyx_t_8[3] = __pyx_t_5;
        __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_8, 4, 23 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 2 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5), 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_5));
        if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 246, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
        __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
        __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
        __pyx_v_err = ((PyObject*)__pyx_t_4);
        __pyx_t_4 = 0;

        /* "cupy/fft/_callback.pyx":247
 *                 # by the current CUDA ver?
 *                 err = f'nvprune failed with sm_{arch}:\n{p.stderr}'
 *                 raise RuntimeError(err)             # <<<<<<<<<<<<<<
 *             else:
 *                 # atomic move with the destination guaranteed to be overwritten
*/
        __pyx_t_5 = NULL;
        __pyx_t_6 = 1;
        {
          PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_v_err};
          __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
          if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 247, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_4);
        }
        __Pyx_Raise(__pyx_t_4, 0, 0, 0);
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        __PYX_ERR(0, 247, __pyx_L1_error)

        /* "cupy/fft/_callback.pyx":243
 *                                stderr=subprocess.PIPE)
 *             p.check_returncode()
 *             if p.stderr:             # <<<<<<<<<<<<<<
 *                 # maybe this is a new arch that we haven't seen or unsupported
 *                 # by the current CUDA ver?
*/
      }

      /* "cupy/fft/_callback.pyx":251
 *                 # atomic move with the destination guaranteed to be overwritten
 *                 # (using os.replace() is also ok here)
 *                 os.rename(cufft_lib_temp, cufft_lib_cached)             # <<<<<<<<<<<<<<
 *     else:
 *         # nvprune is not found, just link against the full static lib
*/
      /*else*/ {
        __pyx_t_5 = NULL;
        __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 251, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
        __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_rename); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 251, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_14);
        __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
        __pyx_t_6 = 1;
        #if CYTHON_UNPACK_METHODS
        if (unlikely(PyMethod_Check(__pyx_t_14))) {
          __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_14);
          assert(__pyx_t_5);
          PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14);
          __Pyx_INCREF(__pyx_t_5);
          __Pyx_INCREF(__pyx__function);
          __Pyx_DECREF_SET(__pyx_t_14, __pyx__function);
          __pyx_t_6 = 0;
        }
        #endif
        {
          PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_cufft_lib_temp, __pyx_v_cufft_lib_cached};
          __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
          __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
          if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 251, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_4);
        }
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      }

      /* "cupy/fft/_callback.pyx":234
 *         cufft_lib_cached = os.path.join(cache_dir,
 *                                         'lib' + cufft_lib_pruned + '.a')
 *         if not os.path.isfile(cufft_lib_cached):             # <<<<<<<<<<<<<<
 *             comm = [_nvprune]
 *             for cc in _cc_major_map[arch[:-1]]:
*/
    }

    /* "cupy/fft/_callback.pyx":227
 *     cdef str cufft_lib_full, cufft_lib_pruned, cufft_lib_temp, cufft_lib_cached
 * 
 *     if _nvprune:             # <<<<<<<<<<<<<<
 *         cufft_lib_full = os.path.join(_cuda_path, 'lib64/libcufft_static.a')
 *         cufft_lib_pruned = f'cufft_static_{_cufft_ver}_sm{arch[0]}'
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":254
 *     else:
 *         # nvprune is not found, just link against the full static lib
 *         cufft_lib_pruned = None             # <<<<<<<<<<<<<<
 * 
 *     return cufft_lib_pruned
*/
  /*else*/ {
    __Pyx_INCREF(Py_None);
    __pyx_v_cufft_lib_pruned = ((PyObject*)Py_None);
  }
  __pyx_L3:;

  /* "cupy/fft/_callback.pyx":256
 *         cufft_lib_pruned = None
 * 
 *     return cufft_lib_pruned             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_cufft_lib_pruned);
  __pyx_r = __pyx_v_cufft_lib_pruned;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":224
 * 
 * 
 * cdef inline str _prune(str temp_dir, str cache_dir, str _cufft_ver, str arch):             # <<<<<<<<<<<<<<
 *     cdef str cufft_lib_full, cufft_lib_pruned, cufft_lib_temp, cufft_lib_cached
 * 
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_XDECREF(__pyx_t_14);
  __Pyx_AddTraceback("cupy.fft._callback._prune", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_cufft_lib_full);
  __Pyx_XDECREF(__pyx_v_cufft_lib_pruned);
  __Pyx_XDECREF(__pyx_v_cufft_lib_temp);
  __Pyx_XDECREF(__pyx_v_cufft_lib_cached);
  __Pyx_XDECREF(__pyx_v_comm);
  __Pyx_XDECREF(__pyx_v_cc);
  __Pyx_XDECREF(__pyx_v_p);
  __Pyx_XDECREF(__pyx_v_err);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":259
 * 
 * 
 * cdef inline void _nvcc_compile(             # <<<<<<<<<<<<<<
 *         str tempdir, str mod_name, str cb_load, str cb_store,
 *         str obj_dev, str arch) except*:
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__nvcc_compile(PyObject *__pyx_v_tempdir, CYTHON_UNUSED PyObject *__pyx_v_mod_name, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, PyObject *__pyx_v_obj_dev, PyObject *__pyx_v_arch) {
  PyObject *__pyx_v_support = 0;
  PyObject *__pyx_v_cmd = 0;
  PyObject *__pyx_v_f = NULL;
  PyObject *__pyx_v_p = NULL;
  PyObject *__pyx_v_e = NULL;
  PyObject *__pyx_v_cex = NULL;
  PyObject *__pyx_v_dump = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  size_t __pyx_t_8;
  PyObject *__pyx_t_9 = NULL;
  PyObject *__pyx_t_10 = NULL;
  PyObject *__pyx_t_11 = NULL;
  PyObject *__pyx_t_12 = NULL;
  PyObject *__pyx_t_13 = NULL;
  int __pyx_t_14;
  int __pyx_t_15;
  PyObject *__pyx_t_16 = NULL;
  PyObject *__pyx_t_17 = NULL;
  int __pyx_t_18;
  PyObject *__pyx_t_19 = NULL;
  PyObject *__pyx_t_20 = NULL;
  PyObject *__pyx_t_21 = NULL;
  int __pyx_t_22;
  char const *__pyx_t_23;
  PyObject *__pyx_t_24 = NULL;
  PyObject *__pyx_t_25 = NULL;
  PyObject *__pyx_t_26 = NULL;
  PyObject *__pyx_t_27 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_nvcc_compile", 0);

  /* "cupy/fft/_callback.pyx":266
 *     global _callback_dev_code
 * 
 *     if _callback_dev_code is None:             # <<<<<<<<<<<<<<
 *         with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:
 *             support = _callback_dev_code = f.read()
*/
  __pyx_t_1 = (__pyx_v_4cupy_3fft_9_callback__callback_dev_code == ((PyObject*)Py_None));
  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":267
 * 
 *     if _callback_dev_code is None:
 *         with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:             # <<<<<<<<<<<<<<
 *             support = _callback_dev_code = f.read()
 *     else:
*/
    /*with:*/ {
      __pyx_t_3 = NULL;
      __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 267, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_6);
      __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 267, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_7);
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      __pyx_t_5 = __pyx_t_7;
      __Pyx_INCREF(__pyx_t_5);
      __pyx_t_8 = 0;
      {
        PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_v_4cupy_3fft_9_callback__source_dir, __pyx_mstate_global->__pyx_kp_u_cupy_cufftXt_cu};
        __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
        __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
        if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 267, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
      }
      __pyx_t_8 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_4};
        __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_open, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 267, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
      }
      __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 267, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_9);
      __pyx_t_3 = NULL;
      __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 267, __pyx_L4_error)
      __Pyx_GOTREF(__pyx_t_7);
      __pyx_t_8 = 1;
      #if CYTHON_UNPACK_METHODS
      if (likely(PyMethod_Check(__pyx_t_7))) {
        __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_7);
        assert(__pyx_t_3);
        PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7);
        __Pyx_INCREF(__pyx_t_3);
        __Pyx_INCREF(__pyx__function);
        __Pyx_DECREF_SET(__pyx_t_7, __pyx__function);
        __pyx_t_8 = 0;
      }
      #endif
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL};
        __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
        __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
        if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 267, __pyx_L4_error)
        __Pyx_GOTREF(__pyx_t_4);
      }
      __pyx_t_7 = __pyx_t_4;
      __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      /*try:*/ {
        {
          __Pyx_PyThreadState_declare
          __Pyx_PyThreadState_assign
          __Pyx_ExceptionSave(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12);
          __Pyx_XGOTREF(__pyx_t_10);
          __Pyx_XGOTREF(__pyx_t_11);
          __Pyx_XGOTREF(__pyx_t_12);
          /*try:*/ {
            __pyx_v_f = __pyx_t_7;
            __pyx_t_7 = 0;

            /* "cupy/fft/_callback.pyx":268
 *     if _callback_dev_code is None:
 *         with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:
 *             support = _callback_dev_code = f.read()             # <<<<<<<<<<<<<<
 *     else:
 *         support = _callback_dev_code
*/
            __pyx_t_2 = __pyx_v_f;
            __Pyx_INCREF(__pyx_t_2);
            __pyx_t_8 = 0;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL};
              __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_read, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
              if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 268, __pyx_L8_error)
              __Pyx_GOTREF(__pyx_t_7);
            }
            if (!(likely(PyUnicode_CheckExact(__pyx_t_7))||((__pyx_t_7) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_7))) __PYX_ERR(0, 268, __pyx_L8_error)
            __Pyx_INCREF(__pyx_t_7);
            __pyx_v_support = ((PyObject*)__pyx_t_7);
            __Pyx_INCREF(__pyx_t_7);
            __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__callback_dev_code);
            __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__callback_dev_code, ((PyObject*)__pyx_t_7));
            __Pyx_GIVEREF(__pyx_t_7);
            __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

            /* "cupy/fft/_callback.pyx":267
 * 
 *     if _callback_dev_code is None:
 *         with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:             # <<<<<<<<<<<<<<
 *             support = _callback_dev_code = f.read()
 *     else:
*/
          }
          __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
          __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0;
          __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0;
          goto __pyx_L13_try_end;
          __pyx_L8_error:;
          __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
          __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
          __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
          __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
          /*except:*/ {
            __Pyx_AddTraceback("cupy.fft._callback._nvcc_compile", __pyx_clineno, __pyx_lineno, __pyx_filename);
            if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_4) < 0) __PYX_ERR(0, 267, __pyx_L10_except_error)
            __Pyx_XGOTREF(__pyx_t_7);
            __Pyx_XGOTREF(__pyx_t_2);
            __Pyx_XGOTREF(__pyx_t_4);
            __pyx_t_3 = PyTuple_Pack(3, __pyx_t_7, __pyx_t_2, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 267, __pyx_L10_except_error)
            __Pyx_GOTREF(__pyx_t_3);
            __pyx_t_13 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_3, NULL);
            __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
            __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
            if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 267, __pyx_L10_except_error)
            __Pyx_GOTREF(__pyx_t_13);
            __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_13);
            __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
            if (__pyx_t_1 < (0)) __PYX_ERR(0, 267, __pyx_L10_except_error)
            __pyx_t_14 = (!__pyx_t_1);
            if (unlikely(__pyx_t_14)) {
              __Pyx_GIVEREF(__pyx_t_7);
              __Pyx_GIVEREF(__pyx_t_2);
              __Pyx_XGIVEREF(__pyx_t_4);
              __Pyx_ErrRestoreWithState(__pyx_t_7, __pyx_t_2, __pyx_t_4);
              __pyx_t_7 = 0;  __pyx_t_2 = 0;  __pyx_t_4 = 0; 
              __PYX_ERR(0, 267, __pyx_L10_except_error)
            }
            __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
            __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
            __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
            goto __pyx_L9_exception_handled;
          }
          __pyx_L10_except_error:;
          __Pyx_XGIVEREF(__pyx_t_10);
          __Pyx_XGIVEREF(__pyx_t_11);
          __Pyx_XGIVEREF(__pyx_t_12);
          __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12);
          goto __pyx_L1_error;
          __pyx_L9_exception_handled:;
          __Pyx_XGIVEREF(__pyx_t_10);
          __Pyx_XGIVEREF(__pyx_t_11);
          __Pyx_XGIVEREF(__pyx_t_12);
          __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12);
          __pyx_L13_try_end:;
        }
      }
      /*finally:*/ {
        /*normal exit:*/{
          if (__pyx_t_9) {
            __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_mstate_global->__pyx_tuple[3], NULL);
            __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
            if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 267, __pyx_L1_error)
            __Pyx_GOTREF(__pyx_t_12);
            __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
          }
          goto __pyx_L7;
        }
        __pyx_L7:;
      }
      goto __pyx_L17;
      __pyx_L4_error:;
      __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
      goto __pyx_L1_error;
      __pyx_L17:;
    }

    /* "cupy/fft/_callback.pyx":266
 *     global _callback_dev_code
 * 
 *     if _callback_dev_code is None:             # <<<<<<<<<<<<<<
 *         with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:
 *             support = _callback_dev_code = f.read()
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":270
 *             support = _callback_dev_code = f.read()
 *     else:
 *         support = _callback_dev_code             # <<<<<<<<<<<<<<
 *     with open(os.path.join(tempdir, 'cupy_cufftXt.cu'), 'w') as f:
 *         support = string.Template(support).substitute(
*/
  /*else*/ {
    __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__callback_dev_code);
    __pyx_v_support = __pyx_v_4cupy_3fft_9_callback__callback_dev_code;
  }
  __pyx_L3:;

  /* "cupy/fft/_callback.pyx":271
 *     else:
 *         support = _callback_dev_code
 *     with open(os.path.join(tempdir, 'cupy_cufftXt.cu'), 'w') as f:             # <<<<<<<<<<<<<<
 *         support = string.Template(support).substitute(
 *             dev_load_callback_ker=cb_load,
*/
  /*with:*/ {
    __pyx_t_2 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 271, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 271, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    __pyx_t_3 = __pyx_t_6;
    __Pyx_INCREF(__pyx_t_3);
    __pyx_t_8 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_v_tempdir, __pyx_mstate_global->__pyx_kp_u_cupy_cufftXt_cu};
      __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 271, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_7);
    }
    __pyx_t_8 = 1;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_t_7, __pyx_mstate_global->__pyx_n_u_w};
      __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_open, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 271, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 271, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_9);
    __pyx_t_2 = NULL;
    __pyx_t_6 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 271, __pyx_L18_error)
    __Pyx_GOTREF(__pyx_t_6);
    __pyx_t_8 = 1;
    #if CYTHON_UNPACK_METHODS
    if (likely(PyMethod_Check(__pyx_t_6))) {
      __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_6);
      assert(__pyx_t_2);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6);
      __Pyx_INCREF(__pyx_t_2);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_6, __pyx__function);
      __pyx_t_8 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL};
      __pyx_t_7 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 271, __pyx_L18_error)
      __Pyx_GOTREF(__pyx_t_7);
    }
    __pyx_t_6 = __pyx_t_7;
    __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    /*try:*/ {
      {
        __Pyx_PyThreadState_declare
        __Pyx_PyThreadState_assign
        __Pyx_ExceptionSave(&__pyx_t_12, &__pyx_t_11, &__pyx_t_10);
        __Pyx_XGOTREF(__pyx_t_12);
        __Pyx_XGOTREF(__pyx_t_11);
        __Pyx_XGOTREF(__pyx_t_10);
        /*try:*/ {
          __Pyx_XDECREF_SET(__pyx_v_f, __pyx_t_6);
          __pyx_t_6 = 0;

          /* "cupy/fft/_callback.pyx":272
 *         support = _callback_dev_code
 *     with open(os.path.join(tempdir, 'cupy_cufftXt.cu'), 'w') as f:
 *         support = string.Template(support).substitute(             # <<<<<<<<<<<<<<
 *             dev_load_callback_ker=cb_load,
 *             dev_store_callback_ker=cb_store)
*/
          __pyx_t_2 = NULL;
          __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_string); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 272, __pyx_L22_error)
          __Pyx_GOTREF(__pyx_t_3);
          __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_Template); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 272, __pyx_L22_error)
          __Pyx_GOTREF(__pyx_t_5);
          __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
          if (unlikely(!__pyx_v_support)) { __Pyx_RaiseUnboundLocalError("support"); __PYX_ERR(0, 272, __pyx_L22_error) }
          __pyx_t_8 = 1;
          #if CYTHON_UNPACK_METHODS
          if (unlikely(PyMethod_Check(__pyx_t_5))) {
            __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5);
            assert(__pyx_t_2);
            PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5);
            __Pyx_INCREF(__pyx_t_2);
            __Pyx_INCREF(__pyx__function);
            __Pyx_DECREF_SET(__pyx_t_5, __pyx__function);
            __pyx_t_8 = 0;
          }
          #endif
          {
            PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_support};
            __pyx_t_7 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
            __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
            __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
            if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 272, __pyx_L22_error)
            __Pyx_GOTREF(__pyx_t_7);
          }
          __pyx_t_4 = __pyx_t_7;
          __Pyx_INCREF(__pyx_t_4);

          /* "cupy/fft/_callback.pyx":274
 *         support = string.Template(support).substitute(
 *             dev_load_callback_ker=cb_load,
 *             dev_store_callback_ker=cb_store)             # <<<<<<<<<<<<<<
 *         f.write(support)
 *     cmd = _nvcc + ['-arch=sm_'+arch, '-dc',
*/
          __pyx_t_8 = 0;
          {
            PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL};
            __pyx_t_5 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 272, __pyx_L22_error)
            __Pyx_GOTREF(__pyx_t_5);
            if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dev_load_callback_ker, __pyx_v_cb_load, __pyx_t_5, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 272, __pyx_L22_error)
            if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dev_store_callback_ker, __pyx_v_cb_store, __pyx_t_5, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 272, __pyx_L22_error)
            __pyx_t_6 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_substitute, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_5);
            __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
            __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
            __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
            if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 272, __pyx_L22_error)
            __Pyx_GOTREF(__pyx_t_6);
          }

          /* "cupy/fft/_callback.pyx":272
 *         support = _callback_dev_code
 *     with open(os.path.join(tempdir, 'cupy_cufftXt.cu'), 'w') as f:
 *         support = string.Template(support).substitute(             # <<<<<<<<<<<<<<
 *             dev_load_callback_ker=cb_load,
 *             dev_store_callback_ker=cb_store)
*/
          if (!(likely(PyUnicode_CheckExact(__pyx_t_6))||((__pyx_t_6) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_6))) __PYX_ERR(0, 272, __pyx_L22_error)
          __Pyx_XDECREF_SET(__pyx_v_support, ((PyObject*)__pyx_t_6));
          __pyx_t_6 = 0;

          /* "cupy/fft/_callback.pyx":275
 *             dev_load_callback_ker=cb_load,
 *             dev_store_callback_ker=cb_store)
 *         f.write(support)             # <<<<<<<<<<<<<<
 *     cmd = _nvcc + ['-arch=sm_'+arch, '-dc',
 *                    '-I' + _cupy_include,
*/
          __pyx_t_7 = __pyx_v_f;
          __Pyx_INCREF(__pyx_t_7);
          __pyx_t_8 = 0;
          {
            PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_v_support};
            __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_write, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
            __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
            if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 275, __pyx_L22_error)
            __Pyx_GOTREF(__pyx_t_6);
          }
          __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;

          /* "cupy/fft/_callback.pyx":271
 *     else:
 *         support = _callback_dev_code
 *     with open(os.path.join(tempdir, 'cupy_cufftXt.cu'), 'w') as f:             # <<<<<<<<<<<<<<
 *         support = string.Template(support).substitute(
 *             dev_load_callback_ker=cb_load,
*/
        }
        __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0;
        __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0;
        __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
        goto __pyx_L27_try_end;
        __pyx_L22_error:;
        __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
        __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
        __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
        __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
        __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
        __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
        /*except:*/ {
          __Pyx_AddTraceback("cupy.fft._callback._nvcc_compile", __pyx_clineno, __pyx_lineno, __pyx_filename);
          if (__Pyx_GetException(&__pyx_t_6, &__pyx_t_7, &__pyx_t_5) < 0) __PYX_ERR(0, 271, __pyx_L24_except_error)
          __Pyx_XGOTREF(__pyx_t_6);
          __Pyx_XGOTREF(__pyx_t_7);
          __Pyx_XGOTREF(__pyx_t_5);
          __pyx_t_4 = PyTuple_Pack(3, __pyx_t_6, __pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 271, __pyx_L24_except_error)
          __Pyx_GOTREF(__pyx_t_4);
          __pyx_t_13 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_4, NULL);
          __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
          if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 271, __pyx_L24_except_error)
          __Pyx_GOTREF(__pyx_t_13);
          __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_13);
          __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
          if (__pyx_t_14 < (0)) __PYX_ERR(0, 271, __pyx_L24_except_error)
          __pyx_t_1 = (!__pyx_t_14);
          if (unlikely(__pyx_t_1)) {
            __Pyx_GIVEREF(__pyx_t_6);
            __Pyx_GIVEREF(__pyx_t_7);
            __Pyx_XGIVEREF(__pyx_t_5);
            __Pyx_ErrRestoreWithState(__pyx_t_6, __pyx_t_7, __pyx_t_5);
            __pyx_t_6 = 0;  __pyx_t_7 = 0;  __pyx_t_5 = 0; 
            __PYX_ERR(0, 271, __pyx_L24_except_error)
          }
          __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
          __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
          __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
          goto __pyx_L23_exception_handled;
        }
        __pyx_L24_except_error:;
        __Pyx_XGIVEREF(__pyx_t_12);
        __Pyx_XGIVEREF(__pyx_t_11);
        __Pyx_XGIVEREF(__pyx_t_10);
        __Pyx_ExceptionReset(__pyx_t_12, __pyx_t_11, __pyx_t_10);
        goto __pyx_L1_error;
        __pyx_L23_exception_handled:;
        __Pyx_XGIVEREF(__pyx_t_12);
        __Pyx_XGIVEREF(__pyx_t_11);
        __Pyx_XGIVEREF(__pyx_t_10);
        __Pyx_ExceptionReset(__pyx_t_12, __pyx_t_11, __pyx_t_10);
        __pyx_L27_try_end:;
      }
    }
    /*finally:*/ {
      /*normal exit:*/{
        if (__pyx_t_9) {
          __pyx_t_10 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_mstate_global->__pyx_tuple[3], NULL);
          __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
          if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 271, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_10);
          __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
        }
        goto __pyx_L21;
      }
      __pyx_L21:;
    }
    goto __pyx_L31;
    __pyx_L18_error:;
    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
    goto __pyx_L1_error;
    __pyx_L31:;
  }

  /* "cupy/fft/_callback.pyx":276
 *             dev_store_callback_ker=cb_store)
 *         f.write(support)
 *     cmd = _nvcc + ['-arch=sm_'+arch, '-dc',             # <<<<<<<<<<<<<<
 *                    '-I' + _cupy_include,
 *                    '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),
*/
  __pyx_t_5 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_arch_sm, __pyx_v_arch); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 276, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);

  /* "cupy/fft/_callback.pyx":277
 *         f.write(support)
 *     cmd = _nvcc + ['-arch=sm_'+arch, '-dc',
 *                    '-I' + _cupy_include,             # <<<<<<<<<<<<<<
 *                    '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *                    '-Xcompiler', '-fPIC', '-O2', '-std=c++17']
*/
  __pyx_t_7 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_I, __pyx_v_4cupy_3fft_9_callback__cupy_include); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 277, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);

  /* "cupy/fft/_callback.pyx":278
 *     cmd = _nvcc + ['-arch=sm_'+arch, '-dc',
 *                    '-I' + _cupy_include,
 *                    '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),             # <<<<<<<<<<<<<<
 *                    '-Xcompiler', '-fPIC', '-O2', '-std=c++17']
 *     if cb_load:
*/
  __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 278, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 278, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  __pyx_t_4 = __pyx_t_3;
  __Pyx_INCREF(__pyx_t_4);
  __pyx_t_8 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_tempdir, __pyx_mstate_global->__pyx_kp_u_cupy_cufftXt_cu};
    __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 278, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
  }

  /* "cupy/fft/_callback.pyx":276
 *             dev_store_callback_ker=cb_store)
 *         f.write(support)
 *     cmd = _nvcc + ['-arch=sm_'+arch, '-dc',             # <<<<<<<<<<<<<<
 *                    '-I' + _cupy_include,
 *                    '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),
*/
  __pyx_t_3 = PyList_New(9); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 276, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_GIVEREF(__pyx_t_5);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_dc);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_dc);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_kp_u_dc) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_7);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 2, __pyx_t_7) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_c);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_c);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 3, __pyx_mstate_global->__pyx_kp_u_c) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_6);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 4, __pyx_t_6) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_Xcompiler);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_Xcompiler);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 5, __pyx_mstate_global->__pyx_kp_u_Xcompiler) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_fPIC);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_fPIC);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 6, __pyx_mstate_global->__pyx_kp_u_fPIC) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_O2);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_O2);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 7, __pyx_mstate_global->__pyx_kp_u_O2) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_std_c_17);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_std_c_17);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 8, __pyx_mstate_global->__pyx_kp_u_std_c_17) != (0)) __PYX_ERR(0, 276, __pyx_L1_error);
  __pyx_t_5 = 0;
  __pyx_t_7 = 0;
  __pyx_t_6 = 0;
  __pyx_t_6 = PyNumber_Add(__pyx_v_4cupy_3fft_9_callback__nvcc, __pyx_t_3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 276, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __pyx_v_cmd = ((PyObject*)__pyx_t_6);
  __pyx_t_6 = 0;

  /* "cupy/fft/_callback.pyx":280
 *                    '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *                    '-Xcompiler', '-fPIC', '-O2', '-std=c++17']
 *     if cb_load:             # <<<<<<<<<<<<<<
 *         cmd.append('-DHAS_LOAD_CALLBACK')
 *     if cb_store:
*/
  if (__pyx_v_cb_load == Py_None) __pyx_t_1 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_load);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 280, __pyx_L1_error)
    __pyx_t_1 = (__pyx_temp != 0);
  }

  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":281
 *                    '-Xcompiler', '-fPIC', '-O2', '-std=c++17']
 *     if cb_load:
 *         cmd.append('-DHAS_LOAD_CALLBACK')             # <<<<<<<<<<<<<<
 *     if cb_store:
 *         cmd.append('-DHAS_STORE_CALLBACK')
*/
    __pyx_t_15 = __Pyx_PyList_Append(__pyx_v_cmd, __pyx_mstate_global->__pyx_kp_u_DHAS_LOAD_CALLBACK); if (unlikely(__pyx_t_15 == ((int)-1))) __PYX_ERR(0, 281, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":280
 *                    '-c', os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *                    '-Xcompiler', '-fPIC', '-O2', '-std=c++17']
 *     if cb_load:             # <<<<<<<<<<<<<<
 *         cmd.append('-DHAS_LOAD_CALLBACK')
 *     if cb_store:
*/
  }

  /* "cupy/fft/_callback.pyx":282
 *     if cb_load:
 *         cmd.append('-DHAS_LOAD_CALLBACK')
 *     if cb_store:             # <<<<<<<<<<<<<<
 *         cmd.append('-DHAS_STORE_CALLBACK')
 *     p = subprocess.run(cmd + ['-o', obj_dev],
*/
  if (__pyx_v_cb_store == Py_None) __pyx_t_1 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cb_store);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 282, __pyx_L1_error)
    __pyx_t_1 = (__pyx_temp != 0);
  }

  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":283
 *         cmd.append('-DHAS_LOAD_CALLBACK')
 *     if cb_store:
 *         cmd.append('-DHAS_STORE_CALLBACK')             # <<<<<<<<<<<<<<
 *     p = subprocess.run(cmd + ['-o', obj_dev],
 *                        env=os.environ, cwd=tempdir,
*/
    __pyx_t_15 = __Pyx_PyList_Append(__pyx_v_cmd, __pyx_mstate_global->__pyx_kp_u_DHAS_STORE_CALLBACK); if (unlikely(__pyx_t_15 == ((int)-1))) __PYX_ERR(0, 283, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":282
 *     if cb_load:
 *         cmd.append('-DHAS_LOAD_CALLBACK')
 *     if cb_store:             # <<<<<<<<<<<<<<
 *         cmd.append('-DHAS_STORE_CALLBACK')
 *     p = subprocess.run(cmd + ['-o', obj_dev],
*/
  }

  /* "cupy/fft/_callback.pyx":284
 *     if cb_store:
 *         cmd.append('-DHAS_STORE_CALLBACK')
 *     p = subprocess.run(cmd + ['-o', obj_dev],             # <<<<<<<<<<<<<<
 *                        env=os.environ, cwd=tempdir,
 *                        stdout=subprocess.PIPE,
*/
  __pyx_t_3 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 284, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_run); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 284, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
  __pyx_t_7 = PyList_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 284, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_o);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_o);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_7, 0, __pyx_mstate_global->__pyx_kp_u_o) != (0)) __PYX_ERR(0, 284, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_obj_dev);
  __Pyx_GIVEREF(__pyx_v_obj_dev);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_7, 1, __pyx_v_obj_dev) != (0)) __PYX_ERR(0, 284, __pyx_L1_error);
  __pyx_t_4 = PyNumber_Add(__pyx_v_cmd, __pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 284, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

  /* "cupy/fft/_callback.pyx":285
 *         cmd.append('-DHAS_STORE_CALLBACK')
 *     p = subprocess.run(cmd + ['-o', obj_dev],
 *                        env=os.environ, cwd=tempdir,             # <<<<<<<<<<<<<<
 *                        stdout=subprocess.PIPE,
 *                        stderr=subprocess.PIPE)
*/
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 285, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 285, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

  /* "cupy/fft/_callback.pyx":286
 *     p = subprocess.run(cmd + ['-o', obj_dev],
 *                        env=os.environ, cwd=tempdir,
 *                        stdout=subprocess.PIPE,             # <<<<<<<<<<<<<<
 *                        stderr=subprocess.PIPE)
 *     try:
*/
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 286, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_16 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_PIPE); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 286, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_16);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

  /* "cupy/fft/_callback.pyx":287
 *                        env=os.environ, cwd=tempdir,
 *                        stdout=subprocess.PIPE,
 *                        stderr=subprocess.PIPE)             # <<<<<<<<<<<<<<
 *     try:
 *         p.check_returncode()
*/
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 287, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_17 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_PIPE); if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 287, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_17);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
  __pyx_t_8 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_5))) {
    __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_5);
    assert(__pyx_t_3);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5);
    __Pyx_INCREF(__pyx_t_3);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_5, __pyx__function);
    __pyx_t_8 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 4 : 0)] = {__pyx_t_3, __pyx_t_4};
    __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 284, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_env, __pyx_t_2, __pyx_t_7, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 284, __pyx_L1_error)
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cwd, __pyx_v_tempdir, __pyx_t_7, __pyx_callargs+2, 1) < (0)) __PYX_ERR(0, 284, __pyx_L1_error)
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_stdout, __pyx_t_16, __pyx_t_7, __pyx_callargs+2, 2) < (0)) __PYX_ERR(0, 284, __pyx_L1_error)
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_stderr, __pyx_t_17, __pyx_t_7, __pyx_callargs+2, 3) < (0)) __PYX_ERR(0, 284, __pyx_L1_error)
    __pyx_t_6 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7);
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0;
    __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 284, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
  }
  __pyx_v_p = __pyx_t_6;
  __pyx_t_6 = 0;

  /* "cupy/fft/_callback.pyx":288
 *                        stdout=subprocess.PIPE,
 *                        stderr=subprocess.PIPE)
 *     try:             # <<<<<<<<<<<<<<
 *         p.check_returncode()
 *     except subprocess.CalledProcessError as e:
*/
  {
    __Pyx_PyThreadState_declare
    __Pyx_PyThreadState_assign
    __Pyx_ExceptionSave(&__pyx_t_9, &__pyx_t_10, &__pyx_t_11);
    __Pyx_XGOTREF(__pyx_t_9);
    __Pyx_XGOTREF(__pyx_t_10);
    __Pyx_XGOTREF(__pyx_t_11);
    /*try:*/ {

      /* "cupy/fft/_callback.pyx":289
 *                        stderr=subprocess.PIPE)
 *     try:
 *         p.check_returncode()             # <<<<<<<<<<<<<<
 *     except subprocess.CalledProcessError as e:
 *         cex = CompileException(
*/
      __pyx_t_5 = __pyx_v_p;
      __Pyx_INCREF(__pyx_t_5);
      __pyx_t_8 = 0;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_5, NULL};
        __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_check_returncode, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
        if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 289, __pyx_L34_error)
        __Pyx_GOTREF(__pyx_t_6);
      }
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;

      /* "cupy/fft/_callback.pyx":288
 *                        stdout=subprocess.PIPE,
 *                        stderr=subprocess.PIPE)
 *     try:             # <<<<<<<<<<<<<<
 *         p.check_returncode()
 *     except subprocess.CalledProcessError as e:
*/
    }
    __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0;
    __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
    __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0;
    goto __pyx_L39_try_end;
    __pyx_L34_error:;
    __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0;
    __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0;
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;

    /* "cupy/fft/_callback.pyx":290
 *     try:
 *         p.check_returncode()
 *     except subprocess.CalledProcessError as e:             # <<<<<<<<<<<<<<
 *         cex = CompileException(
 *             str(e) + '\nStderr: ' + e.stderr.decode(), support,
*/
    __Pyx_ErrFetch(&__pyx_t_6, &__pyx_t_5, &__pyx_t_7);
    __Pyx_GetModuleGlobalName(__pyx_t_17, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 290, __pyx_L36_except_error)
    __Pyx_GOTREF(__pyx_t_17);
    __pyx_t_16 = __Pyx_PyObject_GetAttrStr(__pyx_t_17, __pyx_mstate_global->__pyx_n_u_CalledProcessError); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 290, __pyx_L36_except_error)
    __Pyx_GOTREF(__pyx_t_16);
    __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0;
    __pyx_t_18 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_6, __pyx_t_16);
    __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0;
    __Pyx_ErrRestore(__pyx_t_6, __pyx_t_5, __pyx_t_7);
    __pyx_t_6 = 0; __pyx_t_5 = 0; __pyx_t_7 = 0;
    if (__pyx_t_18) {
      __Pyx_AddTraceback("cupy.fft._callback._nvcc_compile", __pyx_clineno, __pyx_lineno, __pyx_filename);
      if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_5, &__pyx_t_6) < 0) __PYX_ERR(0, 290, __pyx_L36_except_error)
      __Pyx_XGOTREF(__pyx_t_7);
      __Pyx_XGOTREF(__pyx_t_5);
      __Pyx_XGOTREF(__pyx_t_6);
      __Pyx_INCREF(__pyx_t_5);
      __pyx_v_e = __pyx_t_5;
      /*try:*/ {

        /* "cupy/fft/_callback.pyx":291
 *         p.check_returncode()
 *     except subprocess.CalledProcessError as e:
 *         cex = CompileException(             # <<<<<<<<<<<<<<
 *             str(e) + '\nStderr: ' + e.stderr.decode(), support,
 *             os.path.join(tempdir, 'cupy_cufftXt.cu'),
*/
        __pyx_t_17 = NULL;
        __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_CompileException); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 291, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_2);

        /* "cupy/fft/_callback.pyx":292
 *     except subprocess.CalledProcessError as e:
 *         cex = CompileException(
 *             str(e) + '\nStderr: ' + e.stderr.decode(), support,             # <<<<<<<<<<<<<<
 *             os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *             cmd[1:], 'nvcc')
*/
        __pyx_t_4 = __Pyx_PyObject_Unicode(__pyx_v_e); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 292, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_4);
        __pyx_t_3 = __Pyx_PyUnicode_Concat__Pyx_ReferenceSharing_OwnStrongReferenceInPlace(__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Stderr); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 292, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_3);
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        __pyx_t_20 = __Pyx_PyObject_GetAttrStr(__pyx_v_e, __pyx_mstate_global->__pyx_n_u_stderr); if (unlikely(!__pyx_t_20)) __PYX_ERR(0, 292, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_20);
        __pyx_t_19 = __pyx_t_20;
        __Pyx_INCREF(__pyx_t_19);
        __pyx_t_8 = 0;
        {
          PyObject *__pyx_callargs[2] = {__pyx_t_19, NULL};
          __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_decode, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_19); __pyx_t_19 = 0;
          __Pyx_DECREF(__pyx_t_20); __pyx_t_20 = 0;
          if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 292, __pyx_L45_error)
          __Pyx_GOTREF(__pyx_t_4);
        }
        __pyx_t_20 = PyNumber_Add(__pyx_t_3, __pyx_t_4); if (unlikely(!__pyx_t_20)) __PYX_ERR(0, 292, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_20);
        __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
        __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_v_support)) { __Pyx_RaiseUnboundLocalError("support"); __PYX_ERR(0, 292, __pyx_L45_error) }

        /* "cupy/fft/_callback.pyx":293
 *         cex = CompileException(
 *             str(e) + '\nStderr: ' + e.stderr.decode(), support,
 *             os.path.join(tempdir, 'cupy_cufftXt.cu'),             # <<<<<<<<<<<<<<
 *             cmd[1:], 'nvcc')
 *         dump = _get_bool_env_variable(
*/
        __Pyx_GetModuleGlobalName(__pyx_t_19, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_19)) __PYX_ERR(0, 293, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_19);
        __pyx_t_21 = __Pyx_PyObject_GetAttrStr(__pyx_t_19, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 293, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_21);
        __Pyx_DECREF(__pyx_t_19); __pyx_t_19 = 0;
        __pyx_t_3 = __pyx_t_21;
        __Pyx_INCREF(__pyx_t_3);
        __pyx_t_8 = 0;
        {
          PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_v_tempdir, __pyx_mstate_global->__pyx_kp_u_cupy_cufftXt_cu};
          __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0;
          if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 293, __pyx_L45_error)
          __Pyx_GOTREF(__pyx_t_4);
        }

        /* "cupy/fft/_callback.pyx":294
 *             str(e) + '\nStderr: ' + e.stderr.decode(), support,
 *             os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *             cmd[1:], 'nvcc')             # <<<<<<<<<<<<<<
 *         dump = _get_bool_env_variable(
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
*/
        __pyx_t_21 = __Pyx_PyList_GetSlice(__pyx_v_cmd, 1, PY_SSIZE_T_MAX); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 294, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_21);
        __pyx_t_8 = 1;
        #if CYTHON_UNPACK_METHODS
        if (unlikely(PyMethod_Check(__pyx_t_2))) {
          __pyx_t_17 = PyMethod_GET_SELF(__pyx_t_2);
          assert(__pyx_t_17);
          PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2);
          __Pyx_INCREF(__pyx_t_17);
          __Pyx_INCREF(__pyx__function);
          __Pyx_DECREF_SET(__pyx_t_2, __pyx__function);
          __pyx_t_8 = 0;
        }
        #endif
        {
          PyObject *__pyx_callargs[6] = {__pyx_t_17, __pyx_t_20, __pyx_v_support, __pyx_t_4, __pyx_t_21, __pyx_mstate_global->__pyx_n_u_nvcc};
          __pyx_t_16 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_8, (6-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0;
          __Pyx_DECREF(__pyx_t_20); __pyx_t_20 = 0;
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
          __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0;
          __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
          if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 291, __pyx_L45_error)
          __Pyx_GOTREF(__pyx_t_16);
        }
        __pyx_v_cex = __pyx_t_16;
        __pyx_t_16 = 0;

        /* "cupy/fft/_callback.pyx":295
 *             os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *             cmd[1:], 'nvcc')
 *         dump = _get_bool_env_variable(             # <<<<<<<<<<<<<<
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
 *         if dump:
*/
        __Pyx_GetModuleGlobalName(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_get_bool_env_variable); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 295, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_16);

        /* "cupy/fft/_callback.pyx":296
 *             cmd[1:], 'nvcc')
 *         dump = _get_bool_env_variable(
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)             # <<<<<<<<<<<<<<
 *         if dump:
 *             cex.dump(sys.stderr)
*/
        __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_16, __pyx_mstate_global->__pyx_tuple[4], NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 295, __pyx_L45_error)
        __Pyx_GOTREF(__pyx_t_2);
        __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0;
        __pyx_v_dump = __pyx_t_2;
        __pyx_t_2 = 0;

        /* "cupy/fft/_callback.pyx":297
 *         dump = _get_bool_env_variable(
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
 *         if dump:             # <<<<<<<<<<<<<<
 *             cex.dump(sys.stderr)
 *         raise cex from e
*/
        __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_dump); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 297, __pyx_L45_error)
        if (__pyx_t_1) {

          /* "cupy/fft/_callback.pyx":298
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
 *         if dump:
 *             cex.dump(sys.stderr)             # <<<<<<<<<<<<<<
 *         raise cex from e
 * 
*/
          __pyx_t_16 = __pyx_v_cex;
          __Pyx_INCREF(__pyx_t_16);
          __Pyx_GetModuleGlobalName(__pyx_t_21, __pyx_mstate_global->__pyx_n_u_sys); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 298, __pyx_L45_error)
          __Pyx_GOTREF(__pyx_t_21);
          __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_21, __pyx_mstate_global->__pyx_n_u_stderr); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 298, __pyx_L45_error)
          __Pyx_GOTREF(__pyx_t_4);
          __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0;
          __pyx_t_8 = 0;
          {
            PyObject *__pyx_callargs[2] = {__pyx_t_16, __pyx_t_4};
            __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_dump, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
            __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0;
            __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
            if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 298, __pyx_L45_error)
            __Pyx_GOTREF(__pyx_t_2);
          }
          __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

          /* "cupy/fft/_callback.pyx":297
 *         dump = _get_bool_env_variable(
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
 *         if dump:             # <<<<<<<<<<<<<<
 *             cex.dump(sys.stderr)
 *         raise cex from e
*/
        }

        /* "cupy/fft/_callback.pyx":299
 *         if dump:
 *             cex.dump(sys.stderr)
 *         raise cex from e             # <<<<<<<<<<<<<<
 * 
 * 
*/
        __Pyx_Raise(__pyx_v_cex, 0, 0, __pyx_v_e);
        __PYX_ERR(0, 299, __pyx_L45_error)
      }

      /* "cupy/fft/_callback.pyx":290
 *     try:
 *         p.check_returncode()
 *     except subprocess.CalledProcessError as e:             # <<<<<<<<<<<<<<
 *         cex = CompileException(
 *             str(e) + '\nStderr: ' + e.stderr.decode(), support,
*/
      /*finally:*/ {
        __pyx_L45_error:;
        /*exception exit:*/{
          __Pyx_PyThreadState_declare
          __Pyx_PyThreadState_assign
          __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_24 = 0; __pyx_t_25 = 0; __pyx_t_26 = 0; __pyx_t_27 = 0;
          __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0;
          __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0;
          __Pyx_XDECREF(__pyx_t_19); __pyx_t_19 = 0;
          __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
          __Pyx_XDECREF(__pyx_t_20); __pyx_t_20 = 0;
          __Pyx_XDECREF(__pyx_t_21); __pyx_t_21 = 0;
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
           __Pyx_ExceptionSwap(&__pyx_t_25, &__pyx_t_26, &__pyx_t_27);
          if ( unlikely(__Pyx_GetException(&__pyx_t_12, &__pyx_t_13, &__pyx_t_24) < 0)) __Pyx_ErrFetch(&__pyx_t_12, &__pyx_t_13, &__pyx_t_24);
          __Pyx_XGOTREF(__pyx_t_12);
          __Pyx_XGOTREF(__pyx_t_13);
          __Pyx_XGOTREF(__pyx_t_24);
          __Pyx_XGOTREF(__pyx_t_25);
          __Pyx_XGOTREF(__pyx_t_26);
          __Pyx_XGOTREF(__pyx_t_27);
          __pyx_t_18 = __pyx_lineno; __pyx_t_22 = __pyx_clineno; __pyx_t_23 = __pyx_filename;
          {
            __Pyx_DECREF(__pyx_v_e); __pyx_v_e = 0;
          }
          __Pyx_XGIVEREF(__pyx_t_25);
          __Pyx_XGIVEREF(__pyx_t_26);
          __Pyx_XGIVEREF(__pyx_t_27);
          __Pyx_ExceptionReset(__pyx_t_25, __pyx_t_26, __pyx_t_27);
          __Pyx_XGIVEREF(__pyx_t_12);
          __Pyx_XGIVEREF(__pyx_t_13);
          __Pyx_XGIVEREF(__pyx_t_24);
          __Pyx_ErrRestore(__pyx_t_12, __pyx_t_13, __pyx_t_24);
          __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_24 = 0; __pyx_t_25 = 0; __pyx_t_26 = 0; __pyx_t_27 = 0;
          __pyx_lineno = __pyx_t_18; __pyx_clineno = __pyx_t_22; __pyx_filename = __pyx_t_23;
          goto __pyx_L36_except_error;
        }
      }
    }
    goto __pyx_L36_except_error;

    /* "cupy/fft/_callback.pyx":288
 *                        stdout=subprocess.PIPE,
 *                        stderr=subprocess.PIPE)
 *     try:             # <<<<<<<<<<<<<<
 *         p.check_returncode()
 *     except subprocess.CalledProcessError as e:
*/
    __pyx_L36_except_error:;
    __Pyx_XGIVEREF(__pyx_t_9);
    __Pyx_XGIVEREF(__pyx_t_10);
    __Pyx_XGIVEREF(__pyx_t_11);
    __Pyx_ExceptionReset(__pyx_t_9, __pyx_t_10, __pyx_t_11);
    goto __pyx_L1_error;
    __pyx_L39_try_end:;
  }

  /* "cupy/fft/_callback.pyx":259
 * 
 * 
 * cdef inline void _nvcc_compile(             # <<<<<<<<<<<<<<
 *         str tempdir, str mod_name, str cb_load, str cb_store,
 *         str obj_dev, str arch) except*:
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_16);
  __Pyx_XDECREF(__pyx_t_17);
  __Pyx_XDECREF(__pyx_t_19);
  __Pyx_XDECREF(__pyx_t_20);
  __Pyx_XDECREF(__pyx_t_21);
  __Pyx_AddTraceback("cupy.fft._callback._nvcc_compile", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_support);
  __Pyx_XDECREF(__pyx_v_cmd);
  __Pyx_XDECREF(__pyx_v_f);
  __Pyx_XDECREF(__pyx_v_p);
  __Pyx_XDECREF(__pyx_v_e);
  __Pyx_XDECREF(__pyx_v_cex);
  __Pyx_XDECREF(__pyx_v_dump);
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":302
 * 
 * 
 * cdef inline void _nvcc_link(             # <<<<<<<<<<<<<<
 *         str tempdir, str obj_host, str obj_dev, str arch, str mod_filename,
 *         str cache_dir, str cufft_lib_pruned) except*:
*/

static CYTHON_INLINE void __pyx_f_4cupy_3fft_9_callback__nvcc_link(PyObject *__pyx_v_tempdir, PyObject *__pyx_v_obj_host, PyObject *__pyx_v_obj_dev, PyObject *__pyx_v_arch, PyObject *__pyx_v_mod_filename, PyObject *__pyx_v_cache_dir, PyObject *__pyx_v_cufft_lib_pruned) {
  PyObject *__pyx_v_cmd = 0;
  PyObject *__pyx_v_mod_temp = 0;
  PyObject *__pyx_v_mod_cached = 0;
  PyObject *__pyx_v_p = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  int __pyx_t_6;
  int __pyx_t_7;
  PyObject *__pyx_t_8 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_nvcc_link", 0);

  /* "cupy/fft/_callback.pyx":309
 *     cdef str mod_temp, mod_cached
 * 
 *     mod_temp = os.path.join(tempdir, mod_filename)             # <<<<<<<<<<<<<<
 *     mod_cached = os.path.join(cache_dir, mod_filename)
 *     cmd = _nvcc + ['-shared', '-arch=sm_'+arch, obj_dev, obj_host,
*/
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 309, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 309, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __pyx_t_2 = __pyx_t_4;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_5 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_tempdir, __pyx_v_mod_filename};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 309, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 309, __pyx_L1_error)
  __pyx_v_mod_temp = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":310
 * 
 *     mod_temp = os.path.join(tempdir, mod_filename)
 *     mod_cached = os.path.join(cache_dir, mod_filename)             # <<<<<<<<<<<<<<
 *     cmd = _nvcc + ['-shared', '-arch=sm_'+arch, obj_dev, obj_host,
 *                    '-Xlinker', '-Bsymbolic']
*/
  __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 310, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 310, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  __pyx_t_4 = __pyx_t_3;
  __Pyx_INCREF(__pyx_t_4);
  __pyx_t_5 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_cache_dir, __pyx_v_mod_filename};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 310, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 310, __pyx_L1_error)
  __pyx_v_mod_cached = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":311
 *     mod_temp = os.path.join(tempdir, mod_filename)
 *     mod_cached = os.path.join(cache_dir, mod_filename)
 *     cmd = _nvcc + ['-shared', '-arch=sm_'+arch, obj_dev, obj_host,             # <<<<<<<<<<<<<<
 *                    '-Xlinker', '-Bsymbolic']
 *     if cufft_lib_pruned:
*/
  __pyx_t_1 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_arch_sm, __pyx_v_arch); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 311, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_3 = PyList_New(6); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 311, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_shared);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_shared);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 0, __pyx_mstate_global->__pyx_kp_u_shared) != (0)) __PYX_ERR(0, 311, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_1);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 311, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_obj_dev);
  __Pyx_GIVEREF(__pyx_v_obj_dev);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 2, __pyx_v_obj_dev) != (0)) __PYX_ERR(0, 311, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_obj_host);
  __Pyx_GIVEREF(__pyx_v_obj_host);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 3, __pyx_v_obj_host) != (0)) __PYX_ERR(0, 311, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_Xlinker);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_Xlinker);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 4, __pyx_mstate_global->__pyx_kp_u_Xlinker) != (0)) __PYX_ERR(0, 311, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_Bsymbolic);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_Bsymbolic);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 5, __pyx_mstate_global->__pyx_kp_u_Bsymbolic) != (0)) __PYX_ERR(0, 311, __pyx_L1_error);
  __pyx_t_1 = 0;
  __pyx_t_1 = PyNumber_Add(__pyx_v_4cupy_3fft_9_callback__nvcc, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 311, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __pyx_v_cmd = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":313
 *     cmd = _nvcc + ['-shared', '-arch=sm_'+arch, obj_dev, obj_host,
 *                    '-Xlinker', '-Bsymbolic']
 *     if cufft_lib_pruned:             # <<<<<<<<<<<<<<
 *         cmd += ['-L'+cache_dir, '-l'+cufft_lib_pruned]
 *     else:
*/
  if (__pyx_v_cufft_lib_pruned == Py_None) __pyx_t_6 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_cufft_lib_pruned);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 313, __pyx_L1_error)
    __pyx_t_6 = (__pyx_temp != 0);
  }

  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":314
 *                    '-Xlinker', '-Bsymbolic']
 *     if cufft_lib_pruned:
 *         cmd += ['-L'+cache_dir, '-l'+cufft_lib_pruned]             # <<<<<<<<<<<<<<
 *     else:
 *         cmd.append('-lcufft_static')
*/
    __pyx_t_1 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_L, __pyx_v_cache_dir); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 314, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __pyx_t_3 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_l, __pyx_v_cufft_lib_pruned); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 314, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __pyx_t_4 = PyList_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyList_SET_ITEM(__pyx_t_4, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 314, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_3);
    if (__Pyx_PyList_SET_ITEM(__pyx_t_4, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 314, __pyx_L1_error);
    __pyx_t_1 = 0;
    __pyx_t_3 = 0;
    __pyx_t_3 = PyNumber_InPlaceAdd(__pyx_v_cmd, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 314, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF_SET(__pyx_v_cmd, ((PyObject*)__pyx_t_3));
    __pyx_t_3 = 0;

    /* "cupy/fft/_callback.pyx":313
 *     cmd = _nvcc + ['-shared', '-arch=sm_'+arch, obj_dev, obj_host,
 *                    '-Xlinker', '-Bsymbolic']
 *     if cufft_lib_pruned:             # <<<<<<<<<<<<<<
 *         cmd += ['-L'+cache_dir, '-l'+cufft_lib_pruned]
 *     else:
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":316
 *         cmd += ['-L'+cache_dir, '-l'+cufft_lib_pruned]
 *     else:
 *         cmd.append('-lcufft_static')             # <<<<<<<<<<<<<<
 *     cmd += ['-lculibos', '-lpthread', '-o', mod_temp]
 * 
*/
  /*else*/ {
    __pyx_t_7 = __Pyx_PyList_Append(__pyx_v_cmd, __pyx_mstate_global->__pyx_kp_u_lcufft_static); if (unlikely(__pyx_t_7 == ((int)-1))) __PYX_ERR(0, 316, __pyx_L1_error)
  }
  __pyx_L3:;

  /* "cupy/fft/_callback.pyx":317
 *     else:
 *         cmd.append('-lcufft_static')
 *     cmd += ['-lculibos', '-lpthread', '-o', mod_temp]             # <<<<<<<<<<<<<<
 * 
 *     p = subprocess.run(cmd, env=os.environ, cwd=tempdir)
*/
  __pyx_t_3 = PyList_New(4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 317, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_lculibos);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_lculibos);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 0, __pyx_mstate_global->__pyx_kp_u_lculibos) != (0)) __PYX_ERR(0, 317, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_lpthread);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_lpthread);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_kp_u_lpthread) != (0)) __PYX_ERR(0, 317, __pyx_L1_error);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_o);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_o);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 2, __pyx_mstate_global->__pyx_kp_u_o) != (0)) __PYX_ERR(0, 317, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_mod_temp);
  __Pyx_GIVEREF(__pyx_v_mod_temp);
  if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 3, __pyx_v_mod_temp) != (0)) __PYX_ERR(0, 317, __pyx_L1_error);
  __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_cmd, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 317, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __Pyx_DECREF_SET(__pyx_v_cmd, ((PyObject*)__pyx_t_4));
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":319
 *     cmd += ['-lculibos', '-lpthread', '-o', mod_temp]
 * 
 *     p = subprocess.run(cmd, env=os.environ, cwd=tempdir)             # <<<<<<<<<<<<<<
 *     p.check_returncode()
 *     # atomic move with the destination guaranteed to be overwritten;
*/
  __pyx_t_3 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_subprocess); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 319, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_run); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 319, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 319, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 319, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_8);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_5 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_2))) {
    __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2);
    assert(__pyx_t_3);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2);
    __Pyx_INCREF(__pyx_t_3);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_2, __pyx__function);
    __pyx_t_5 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_3, __pyx_v_cmd};
    __pyx_t_1 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 319, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_env, __pyx_t_8, __pyx_t_1, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 319, __pyx_L1_error)
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cwd, __pyx_v_tempdir, __pyx_t_1, __pyx_callargs+2, 1) < (0)) __PYX_ERR(0, 319, __pyx_L1_error)
    __pyx_t_4 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_1);
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 319, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
  }
  __pyx_v_p = __pyx_t_4;
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":320
 * 
 *     p = subprocess.run(cmd, env=os.environ, cwd=tempdir)
 *     p.check_returncode()             # <<<<<<<<<<<<<<
 *     # atomic move with the destination guaranteed to be overwritten;
 *     # using os.replace() is also ok here
*/
  __pyx_t_2 = __pyx_v_p;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_5 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL};
    __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_check_returncode, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 320, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
  }
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":323
 *     # atomic move with the destination guaranteed to be overwritten;
 *     # using os.replace() is also ok here
 *     os.rename(mod_temp, mod_cached)             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __pyx_t_2 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 323, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_rename); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 323, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_8);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_5 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_8))) {
    __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_8);
    assert(__pyx_t_2);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8);
    __Pyx_INCREF(__pyx_t_2);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_8, __pyx__function);
    __pyx_t_5 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_mod_temp, __pyx_v_mod_cached};
    __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
    if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 323, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
  }
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":302
 * 
 * 
 * cdef inline void _nvcc_link(             # <<<<<<<<<<<<<<
 *         str tempdir, str obj_host, str obj_dev, str arch, str mod_filename,
 *         str cache_dir, str cufft_lib_pruned) except*:
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_AddTraceback("cupy.fft._callback._nvcc_link", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_cmd);
  __Pyx_XDECREF(__pyx_v_mod_temp);
  __Pyx_XDECREF(__pyx_v_mod_cached);
  __Pyx_XDECREF(__pyx_v_p);
  __Pyx_RefNannyFinishContext();
}

/* "cupy/fft/_callback.pyx":327
 * 
 * # make it a plain Python function so that we can mock-test it
 * def get_cache_dir():             # <<<<<<<<<<<<<<
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_1get_cache_dir(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_get_cache_dir, "get_cache_dir()");
static PyMethodDef __pyx_mdef_4cupy_3fft_9_callback_1get_cache_dir = {"get_cache_dir", (PyCFunction)__pyx_pw_4cupy_3fft_9_callback_1get_cache_dir, METH_NOARGS, __pyx_doc_4cupy_3fft_9_callback_get_cache_dir};
static PyObject *__pyx_pw_4cupy_3fft_9_callback_1get_cache_dir(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("get_cache_dir (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_get_cache_dir(__pyx_self);

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_get_cache_dir(CYTHON_UNUSED PyObject *__pyx_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  PyObject *__pyx_t_8 = NULL;
  size_t __pyx_t_9;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("get_cache_dir", 0);

  /* "cupy/fft/_callback.pyx":329
 * def get_cache_dir():
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:             # <<<<<<<<<<<<<<
 *         _callback_cache_dir = os.environ.get(
 *             'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))
*/
  __pyx_t_1 = (__pyx_v_4cupy_3fft_9_callback__callback_cache_dir == ((PyObject*)Py_None));
  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":330
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:
 *         _callback_cache_dir = os.environ.get(             # <<<<<<<<<<<<<<
 *             'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))
 *     return _callback_cache_dir
*/
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 330, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_environ); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 330, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __pyx_t_3 = __pyx_t_5;
    __Pyx_INCREF(__pyx_t_3);

    /* "cupy/fft/_callback.pyx":331
 *     if _callback_cache_dir is None:
 *         _callback_cache_dir = os.environ.get(
 *             'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))             # <<<<<<<<<<<<<<
 *     return _callback_cache_dir
 * 
*/
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 331, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 331, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_8);
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __pyx_t_6 = __pyx_t_8;
    __Pyx_INCREF(__pyx_t_6);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_mstate_global->__pyx_kp_u_cupy_callback_cache};
      __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_expanduser, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
      __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 331, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_mstate_global->__pyx_n_u_CUPY_CACHE_DIR, __pyx_t_4};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 330, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }

    /* "cupy/fft/_callback.pyx":330
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:
 *         _callback_cache_dir = os.environ.get(             # <<<<<<<<<<<<<<
 *             'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))
 *     return _callback_cache_dir
*/
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 330, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__callback_cache_dir);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__callback_cache_dir, ((PyObject*)__pyx_t_2));
    __Pyx_GIVEREF(__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":329
 * def get_cache_dir():
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:             # <<<<<<<<<<<<<<
 *         _callback_cache_dir = os.environ.get(
 *             'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))
*/
  }

  /* "cupy/fft/_callback.pyx":332
 *         _callback_cache_dir = os.environ.get(
 *             'CUPY_CACHE_DIR', os.path.expanduser('~/.cupy/callback_cache'))
 *     return _callback_cache_dir             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__callback_cache_dir);
  __pyx_r = __pyx_v_4cupy_3fft_9_callback__callback_cache_dir;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":327
 * 
 * # make it a plain Python function so that we can mock-test it
 * def get_cache_dir():             # <<<<<<<<<<<<<<
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_AddTraceback("cupy.fft._callback.get_cache_dir", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":335
 * 
 * 
 * cpdef get_current_callback_manager():             # <<<<<<<<<<<<<<
 *     cdef _ThreadLocal tls = _ThreadLocal.get()
 *     cdef _CallbackManager mgr = tls._current_cufft_callback
*/

static PyObject *__pyx_pw_4cupy_3fft_9_callback_3get_current_callback_manager(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_get_current_callback_manager(CYTHON_UNUSED int __pyx_skip_dispatch) {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_tls = 0;
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_mgr = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("get_current_callback_manager", 0);

  /* "cupy/fft/_callback.pyx":336
 * 
 * cpdef get_current_callback_manager():
 *     cdef _ThreadLocal tls = _ThreadLocal.get()             # <<<<<<<<<<<<<<
 *     cdef _CallbackManager mgr = tls._current_cufft_callback
 *     return mgr
*/
  __pyx_t_1 = ((PyObject *)__pyx_f_4cupy_3fft_9_callback_12_ThreadLocal_get()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 336, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_tls = ((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":337
 * cpdef get_current_callback_manager():
 *     cdef _ThreadLocal tls = _ThreadLocal.get()
 *     cdef _CallbackManager mgr = tls._current_cufft_callback             # <<<<<<<<<<<<<<
 *     return mgr
 * 
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_tls->_current_cufft_callback);
  __Pyx_INCREF(__pyx_t_1);
  __pyx_v_mgr = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":338
 *     cdef _ThreadLocal tls = _ThreadLocal.get()
 *     cdef _CallbackManager mgr = tls._current_cufft_callback
 *     return mgr             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF((PyObject *)__pyx_v_mgr);
  __pyx_r = ((PyObject *)__pyx_v_mgr);
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":335
 * 
 * 
 * cpdef get_current_callback_manager():             # <<<<<<<<<<<<<<
 *     cdef _ThreadLocal tls = _ThreadLocal.get()
 *     cdef _CallbackManager mgr = tls._current_cufft_callback
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.get_current_callback_manager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF((PyObject *)__pyx_v_tls);
  __Pyx_XDECREF((PyObject *)__pyx_v_mgr);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_3get_current_callback_manager(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_2get_current_callback_manager, "get_current_callback_manager()");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_3get_current_callback_manager(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("get_current_callback_manager (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_2get_current_callback_manager(__pyx_self);

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_2get_current_callback_manager(CYTHON_UNUSED PyObject *__pyx_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("get_current_callback_manager", 0);
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = __pyx_f_4cupy_3fft_9_callback_get_current_callback_manager(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 335, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_r = __pyx_t_1;
  __pyx_t_1 = 0;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.get_current_callback_manager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":348
 *         readonly str identity
 * 
 *     cdef set_caller_infos(self,             # <<<<<<<<<<<<<<
 *                           MemoryPointer cb_load_data=None,
 *                           MemoryPointer cb_store_data=None):
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self, struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos *__pyx_optional_args) {

  /* "cupy/fft/_callback.pyx":349
 * 
 *     cdef set_caller_infos(self,
 *                           MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                           MemoryPointer cb_store_data=None):
 *         '''Set the auxiliary arrays to be used by the load/store callbacks.
*/
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None);

  /* "cupy/fft/_callback.pyx":350
 *     cdef set_caller_infos(self,
 *                           MemoryPointer cb_load_data=None,
 *                           MemoryPointer cb_store_data=None):             # <<<<<<<<<<<<<<
 *         '''Set the auxiliary arrays to be used by the load/store callbacks.
 *         Corresponding to the ``callerInfo`` field in cuFFT's callback API.
*/
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None);
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("set_caller_infos", 0);
  if (__pyx_optional_args) {
    if (__pyx_optional_args->__pyx_n > 0) {
      __pyx_v_cb_load_data = __pyx_optional_args->cb_load_data;
      if (__pyx_optional_args->__pyx_n > 1) {
        __pyx_v_cb_store_data = __pyx_optional_args->cb_store_data;
      }
    }
  }

  /* "cupy/fft/_callback.pyx":365
 * 
 *         '''
 *         self.cb_load_data = cb_load_data             # <<<<<<<<<<<<<<
 *         self.cb_store_data = cb_store_data
 * 
*/
  __Pyx_INCREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->cb_load_data);
  __Pyx_DECREF((PyObject *)__pyx_v_self->cb_load_data);
  __pyx_v_self->cb_load_data = __pyx_v_cb_load_data;

  /* "cupy/fft/_callback.pyx":366
 *         '''
 *         self.cb_load_data = cb_load_data
 *         self.cb_store_data = cb_store_data             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_INCREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->cb_store_data);
  __Pyx_DECREF((PyObject *)__pyx_v_self->cb_store_data);
  __pyx_v_self->cb_store_data = __pyx_v_cb_store_data;

  /* "cupy/fft/_callback.pyx":348
 *         readonly str identity
 * 
 *     cdef set_caller_infos(self,             # <<<<<<<<<<<<<<
 *                           MemoryPointer cb_load_data=None,
 *                           MemoryPointer cb_store_data=None):
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":344
 * 
 *     cdef:
 *         readonly MemoryPointer cb_load_data             # <<<<<<<<<<<<<<
 *         readonly MemoryPointer cb_store_data
 *         readonly str identity
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_12cb_load_data_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_12cb_load_data_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_12cb_load_data___get__(((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_12cb_load_data___get__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF((PyObject *)__pyx_v_self->cb_load_data);
  __pyx_r = ((PyObject *)__pyx_v_self->cb_load_data);
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":345
 *     cdef:
 *         readonly MemoryPointer cb_load_data
 *         readonly MemoryPointer cb_store_data             # <<<<<<<<<<<<<<
 *         readonly str identity
 * 
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_13cb_store_data_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_13cb_store_data_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_13cb_store_data___get__(((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_13cb_store_data___get__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF((PyObject *)__pyx_v_self->cb_store_data);
  __pyx_r = ((PyObject *)__pyx_v_self->cb_store_data);
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":346
 *         readonly MemoryPointer cb_load_data
 *         readonly MemoryPointer cb_store_data
 *         readonly str identity             # <<<<<<<<<<<<<<
 * 
 *     cdef set_caller_infos(self,
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_8identity_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_8identity_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_8identity___get__(((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_8identity___get__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->identity);
  __pyx_r = __pyx_v_self->identity;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_1__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_16_CallbackManager___reduce_cython__, "_CallbackManager.__reduce_cython__(self)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_1__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return NULL;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_16_CallbackManager___reduce_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager___reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self) {
  PyObject *__pyx_v_state = 0;
  PyObject *__pyx_v__dict = 0;
  int __pyx_v_use_setstate;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__reduce_cython__", 0);

  /* "(tree fragment)":5
 *     cdef object _dict
 *     cdef bint use_setstate
 *     state = (self.cb_load_data, self.cb_store_data, self.identity)             # <<<<<<<<<<<<<<
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
*/
  __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_INCREF((PyObject *)__pyx_v_self->cb_load_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->cb_load_data);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self->cb_load_data)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF((PyObject *)__pyx_v_self->cb_store_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->cb_store_data);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_self->cb_store_data)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->identity);
  __Pyx_GIVEREF(__pyx_v_self->identity);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_self->identity) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __pyx_v_state = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":6
 *     cdef bint use_setstate
 *     state = (self.cb_load_data, self.cb_store_data, self.identity)
 *     _dict = getattr(self, '__dict__', None)             # <<<<<<<<<<<<<<
 *     if _dict is not None and _dict:
 *         state += (_dict,)
*/
  __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 6, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v__dict = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "(tree fragment)":7
 *     state = (self.cb_load_data, self.cb_store_data, self.identity)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
  __pyx_t_3 = (__pyx_v__dict != Py_None);
  if (__pyx_t_3) {
  } else {
    __pyx_t_2 = __pyx_t_3;
    goto __pyx_L4_bool_binop_done;
  }
  __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(3, 7, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_3;
  __pyx_L4_bool_binop_done:;
  if (__pyx_t_2) {

    /* "(tree fragment)":8
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
 *         state += (_dict,)             # <<<<<<<<<<<<<<
 *         use_setstate = True
 *     else:
*/
    __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(__pyx_v__dict);
    __Pyx_GIVEREF(__pyx_v__dict);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(3, 8, __pyx_L1_error);
    __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "(tree fragment)":9
 *     if _dict is not None and _dict:
 *         state += (_dict,)
 *         use_setstate = True             # <<<<<<<<<<<<<<
 *     else:
 *         use_setstate = self.cb_load_data is not None or self.cb_store_data is not None or self.identity is not None
*/
    __pyx_v_use_setstate = 1;

    /* "(tree fragment)":7
 *     state = (self.cb_load_data, self.cb_store_data, self.identity)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
    goto __pyx_L3;
  }

  /* "(tree fragment)":11
 *         use_setstate = True
 *     else:
 *         use_setstate = self.cb_load_data is not None or self.cb_store_data is not None or self.identity is not None             # <<<<<<<<<<<<<<
 *     if use_setstate:
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, None), state
*/
  /*else*/ {
    __pyx_t_3 = (((PyObject *)__pyx_v_self->cb_load_data) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (((PyObject *)__pyx_v_self->cb_store_data) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->identity != ((PyObject*)Py_None));
    __pyx_t_2 = __pyx_t_3;
    __pyx_L6_bool_binop_done:;
    __pyx_v_use_setstate = __pyx_t_2;
  }
  __pyx_L3:;

  /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.cb_load_data is not None or self.cb_store_data is not None or self.identity is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, None), state
 *     else:
*/
  if (__pyx_v_use_setstate) {

    /* "(tree fragment)":13
 *         use_setstate = self.cb_load_data is not None or self.cb_store_data is not None or self.identity is not None
 *     if use_setstate:
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, None), state             # <<<<<<<<<<<<<<
 *     else:
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, state)
*/
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__CallbackManager); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_265177976);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_265177976);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_265177976) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(Py_None);
    __Pyx_GIVEREF(Py_None);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_GIVEREF(__pyx_t_4);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_4 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_5;
    __pyx_t_5 = 0;
    goto __pyx_L0;

    /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.cb_load_data is not None or self.cb_store_data is not None or self.identity is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, None), state
 *     else:
*/
  }

  /* "(tree fragment)":15
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, None), state
 *     else:
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, state)             # <<<<<<<<<<<<<<
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__CallbackManager__set_state(self, __pyx_state)
*/
  /*else*/ {
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__CallbackManager); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_265177976);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_265177976);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_265177976) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_GIVEREF(__pyx_t_5);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_5 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_4;
    __pyx_t_4 = 0;
    goto __pyx_L0;
  }

  /* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback._CallbackManager.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_state);
  __Pyx_XDECREF(__pyx_v__dict);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__CallbackManager__set_state(self, __pyx_state)
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_3__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_16_CallbackManager_2__setstate_cython__, "_CallbackManager.__setstate_cython__(self, __pyx_state)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_3__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 16, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(3, 16, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(3, 16, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
    }
    __pyx_v___pyx_state = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(3, 16, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._CallbackManager.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_2__setstate_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_v_self), __pyx_v___pyx_state);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_16_CallbackManager_2__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_self, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__setstate_cython__", 0);

  /* "(tree fragment)":17
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, state)
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__CallbackManager__set_state(self, __pyx_state)             # <<<<<<<<<<<<<<
*/
  __pyx_t_1 = __pyx_v___pyx_state;
  __Pyx_INCREF(__pyx_t_1);
  if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(3, 17, __pyx_L1_error)
  if (unlikely(__pyx_t_1 == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
    __PYX_ERR(3, 17, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__CallbackManager__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 17, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__CallbackManager, (type(self), 0xfce4b78, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__CallbackManager__set_state(self, __pyx_state)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_AddTraceback("cupy.fft._callback._CallbackManager.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":376
 *         object mod
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  str cb_load='',
 *                  str cb_store='',
*/

/* Python wrapper */
static int __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static int __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
  PyObject *__pyx_v_cb_load = 0;
  PyObject *__pyx_v_cb_store = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data = 0;
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[4] = {0,0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1;
  #endif
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_cb_load,&__pyx_mstate_global->__pyx_n_u_cb_store,&__pyx_mstate_global->__pyx_n_u_cb_load_data,&__pyx_mstate_global->__pyx_n_u_cb_store_data,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 376, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  4:
        values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  3:
        values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(0, 376, __pyx_L3_error)
      if (!values[0]) values[0] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_kp_u__6));
      if (!values[1]) values[1] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_kp_u__6));

      /* "cupy/fft/_callback.pyx":379
 *                  str cb_load='',
 *                  str cb_store='',
 *                  MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_store_data=None):
 *         if not _is_init:
*/
      if (!values[2]) values[2] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":380
 *                  str cb_store='',
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None):             # <<<<<<<<<<<<<<
 *         if not _is_init:
 *             _set_vars()
*/
      if (!values[3]) values[3] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));
    } else {
      switch (__pyx_nargs) {
        case  4:
        values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  3:
        values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 376, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      if (!values[0]) values[0] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_kp_u__6));
      if (!values[1]) values[1] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_kp_u__6));

      /* "cupy/fft/_callback.pyx":379
 *                  str cb_load='',
 *                  str cb_store='',
 *                  MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_store_data=None):
 *         if not _is_init:
*/
      if (!values[2]) values[2] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":380
 *                  str cb_store='',
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None):             # <<<<<<<<<<<<<<
 *         if not _is_init:
 *             _set_vars()
*/
      if (!values[3]) values[3] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));
    }
    __pyx_v_cb_load = ((PyObject*)values[0]);
    __pyx_v_cb_store = ((PyObject*)values[1]);
    __pyx_v_cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)values[2]);
    __pyx_v_cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)values[3]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__init__", 0, 0, 4, __pyx_nargs); __PYX_ERR(0, 376, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return -1;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load), (&PyUnicode_Type), 1, "cb_load", 1))) __PYX_ERR(0, 377, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store), (&PyUnicode_Type), 1, "cb_store", 1))) __PYX_ERR(0, 378, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load_data), __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer, 1, "cb_load_data", 0))) __PYX_ERR(0, 379, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store_data), __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer, 1, "cb_store_data", 0))) __PYX_ERR(0, 380, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager___init__(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self), __pyx_v_cb_load, __pyx_v_cb_store, __pyx_v_cb_load_data, __pyx_v_cb_store_data);

  /* "cupy/fft/_callback.pyx":376
 *         object mod
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  str cb_load='',
 *                  str cb_store='',
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = -1;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static int __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager___init__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data) {
  PyObject *__pyx_v_arch = 0;
  PyObject *__pyx_v_tempdir = 0;
  PyObject *__pyx_v_obj_host = 0;
  PyObject *__pyx_v_obj_dev = 0;
  PyObject *__pyx_v_mod_name = 0;
  PyObject *__pyx_v_mod_filename = 0;
  PyObject *__pyx_v_cache_dir = 0;
  PyObject *__pyx_v_path = 0;
  PyObject *__pyx_v_cufft_lib_pruned = 0;
  PyObject *__pyx_v_keys = NULL;
  PyObject *__pyx_v_tempdir_obj = NULL;
  PyObject *__pyx_v_spec = NULL;
  PyObject *__pyx_v_module = NULL;
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  PyObject *__pyx_t_7 = NULL;
  PyObject *__pyx_t_8 = NULL;
  size_t __pyx_t_9;
  int __pyx_t_10;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__init__", 0);

  /* "cupy/fft/_callback.pyx":381
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None):
 *         if not _is_init:             # <<<<<<<<<<<<<<
 *             _set_vars()
 *         _sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
*/
  __pyx_t_1 = (!__pyx_v_4cupy_3fft_9_callback__is_init);
  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":382
 *                  MemoryPointer cb_store_data=None):
 *         if not _is_init:
 *             _set_vars()             # <<<<<<<<<<<<<<
 *         _sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
 * 
*/
    __pyx_f_4cupy_3fft_9_callback__set_vars(); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 382, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":381
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None):
 *         if not _is_init:             # <<<<<<<<<<<<<<
 *             _set_vars()
 *         _sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
*/
  }

  /* "cupy/fft/_callback.pyx":383
 *         if not _is_init:
 *             _set_vars()
 *         _sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)             # <<<<<<<<<<<<<<
 * 
 *         self.identity = "legacy"
*/
  __pyx_f_4cupy_3fft_9_callback__sanity_checks(__pyx_v_cb_load, __pyx_v_cb_store, __pyx_v_cb_load_data, __pyx_v_cb_store_data); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 383, __pyx_L1_error)

  /* "cupy/fft/_callback.pyx":385
 *         _sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
 * 
 *         self.identity = "legacy"             # <<<<<<<<<<<<<<
 *         self.cb_load = cb_load
 *         self.cb_store = cb_store
*/
  __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_legacy);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_legacy);
  __Pyx_GOTREF(__pyx_v_self->__pyx_base.identity);
  __Pyx_DECREF(__pyx_v_self->__pyx_base.identity);
  __pyx_v_self->__pyx_base.identity = __pyx_mstate_global->__pyx_n_u_legacy;

  /* "cupy/fft/_callback.pyx":386
 * 
 *         self.identity = "legacy"
 *         self.cb_load = cb_load             # <<<<<<<<<<<<<<
 *         self.cb_store = cb_store
 *         self.cb_load_data = cb_load_data
*/
  __Pyx_INCREF(__pyx_v_cb_load);
  __Pyx_GIVEREF(__pyx_v_cb_load);
  __Pyx_GOTREF(__pyx_v_self->cb_load);
  __Pyx_DECREF(__pyx_v_self->cb_load);
  __pyx_v_self->cb_load = __pyx_v_cb_load;

  /* "cupy/fft/_callback.pyx":387
 *         self.identity = "legacy"
 *         self.cb_load = cb_load
 *         self.cb_store = cb_store             # <<<<<<<<<<<<<<
 *         self.cb_load_data = cb_load_data
 *         self.cb_store_data = cb_store_data
*/
  __Pyx_INCREF(__pyx_v_cb_store);
  __Pyx_GIVEREF(__pyx_v_cb_store);
  __Pyx_GOTREF(__pyx_v_self->cb_store);
  __Pyx_DECREF(__pyx_v_self->cb_store);
  __pyx_v_self->cb_store = __pyx_v_cb_store;

  /* "cupy/fft/_callback.pyx":388
 *         self.cb_load = cb_load
 *         self.cb_store = cb_store
 *         self.cb_load_data = cb_load_data             # <<<<<<<<<<<<<<
 *         self.cb_store_data = cb_store_data
 * 
*/
  __Pyx_INCREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __Pyx_DECREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __pyx_v_self->__pyx_base.cb_load_data = __pyx_v_cb_load_data;

  /* "cupy/fft/_callback.pyx":389
 *         self.cb_store = cb_store
 *         self.cb_load_data = cb_load_data
 *         self.cb_store_data = cb_store_data             # <<<<<<<<<<<<<<
 * 
 *         # Set up some variables...
*/
  __Pyx_INCREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __Pyx_DECREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __pyx_v_self->__pyx_base.cb_store_data = __pyx_v_cb_store_data;

  /* "cupy/fft/_callback.pyx":392
 * 
 *         # Set up some variables...
 *         cdef str arch = get_compute_capability()             # <<<<<<<<<<<<<<
 *         cdef str tempdir
 *         cdef str obj_host
*/
  __pyx_t_2 = __pyx_f_4cupy_4cuda_6device_get_compute_capability(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 392, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_v_arch = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":405
 *         # only depends on the given load/store callbacks and the runtime
 *         # environment
 *         keys = (_cc, _nvcc, arch, _build_ver, _cufft_ver, _cupy_ver,             # <<<<<<<<<<<<<<
 *                 _python_include, cb_load, cb_store)
 *         keys = '%s %s %s %s %s %s %s %s %s' % keys
*/
  __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_4cupy_3fft_9_callback__cufft_ver); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 405, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_cupy_ver); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 405, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);

  /* "cupy/fft/_callback.pyx":406
 *         # environment
 *         keys = (_cc, _nvcc, arch, _build_ver, _cufft_ver, _cupy_ver,
 *                 _python_include, cb_load, cb_store)             # <<<<<<<<<<<<<<
 *         keys = '%s %s %s %s %s %s %s %s %s' % keys
 * 
*/
  __pyx_t_4 = PyTuple_New(9); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 405, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__cc);
  __Pyx_GIVEREF(__pyx_v_4cupy_3fft_9_callback__cc);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_4cupy_3fft_9_callback__cc) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__nvcc);
  __Pyx_GIVEREF(__pyx_v_4cupy_3fft_9_callback__nvcc);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_4cupy_3fft_9_callback__nvcc) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_arch);
  __Pyx_GIVEREF(__pyx_v_arch);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_v_arch) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__build_ver);
  __Pyx_GIVEREF(__pyx_v_4cupy_3fft_9_callback__build_ver);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 3, __pyx_v_4cupy_3fft_9_callback__build_ver) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_2);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 4, __pyx_t_2) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_3);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 5, __pyx_t_3) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__python_include);
  __Pyx_GIVEREF(__pyx_v_4cupy_3fft_9_callback__python_include);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 6, __pyx_v_4cupy_3fft_9_callback__python_include) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_cb_load);
  __Pyx_GIVEREF(__pyx_v_cb_load);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 7, __pyx_v_cb_load) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_cb_store);
  __Pyx_GIVEREF(__pyx_v_cb_store);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 8, __pyx_v_cb_store) != (0)) __PYX_ERR(0, 405, __pyx_L1_error);
  __pyx_t_2 = 0;
  __pyx_t_3 = 0;
  __pyx_v_keys = __pyx_t_4;
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":407
 *         keys = (_cc, _nvcc, arch, _build_ver, _cufft_ver, _cupy_ver,
 *                 _python_include, cb_load, cb_store)
 *         keys = '%s %s %s %s %s %s %s %s %s' % keys             # <<<<<<<<<<<<<<
 * 
 *         # Generate module filename: all modules with the identical callbacks
*/
  __pyx_t_4 = PyUnicode_Format(__pyx_mstate_global->__pyx_kp_u_s_s_s_s_s_s_s_s_s, __pyx_v_keys); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 407, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF_SET(__pyx_v_keys, __pyx_t_4);
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":412
 *         # are considered identical regardless of which plan is actually
 *         # executed at the time of generation
 *         mod_name = 'cupy_callback_'             # <<<<<<<<<<<<<<
 *         mod_name += hashlib.md5(
 *             keys.encode(), usedforsecurity=False).hexdigest()
*/
  __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_cupy_callback);
  __pyx_v_mod_name = __pyx_mstate_global->__pyx_n_u_cupy_callback;

  /* "cupy/fft/_callback.pyx":413
 *         # executed at the time of generation
 *         mod_name = 'cupy_callback_'
 *         mod_name += hashlib.md5(             # <<<<<<<<<<<<<<
 *             keys.encode(), usedforsecurity=False).hexdigest()
 *         mod_name = mod_name.replace('.', '')
*/
  __pyx_t_5 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_hashlib); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 413, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_md5); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 413, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;

  /* "cupy/fft/_callback.pyx":414
 *         mod_name = 'cupy_callback_'
 *         mod_name += hashlib.md5(
 *             keys.encode(), usedforsecurity=False).hexdigest()             # <<<<<<<<<<<<<<
 *         mod_name = mod_name.replace('.', '')
 *         mod_filename = mod_name + _ext_suffix
*/
  __pyx_t_8 = __pyx_v_keys;
  __Pyx_INCREF(__pyx_t_8);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_8, NULL};
    __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_encode, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
    if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 414, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
  }
  __pyx_t_9 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_7))) {
    __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_7);
    assert(__pyx_t_5);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7);
    __Pyx_INCREF(__pyx_t_5);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_7, __pyx__function);
    __pyx_t_9 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, __pyx_t_6};
    __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 413, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_8);
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_usedforsecurity, Py_False, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 413, __pyx_L1_error)
    __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8);
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 413, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_t_3 = __pyx_t_2;
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL};
    __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_hexdigest, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
  }

  /* "cupy/fft/_callback.pyx":413
 *         # executed at the time of generation
 *         mod_name = 'cupy_callback_'
 *         mod_name += hashlib.md5(             # <<<<<<<<<<<<<<
 *             keys.encode(), usedforsecurity=False).hexdigest()
 *         mod_name = mod_name.replace('.', '')
*/
  __pyx_t_2 = PyNumber_InPlaceAdd(__pyx_v_mod_name, __pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 413, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 413, __pyx_L1_error)
  __Pyx_DECREF_SET(__pyx_v_mod_name, ((PyObject*)__pyx_t_2));
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":415
 *         mod_name += hashlib.md5(
 *             keys.encode(), usedforsecurity=False).hexdigest()
 *         mod_name = mod_name.replace('.', '')             # <<<<<<<<<<<<<<
 *         mod_filename = mod_name + _ext_suffix
 * 
*/
  if (unlikely(__pyx_v_mod_name == Py_None)) {
    PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "replace");
    __PYX_ERR(0, 415, __pyx_L1_error)
  }
  __pyx_t_2 = PyUnicode_Replace(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u__2, __pyx_mstate_global->__pyx_kp_u__6, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 415, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF_SET(__pyx_v_mod_name, ((PyObject*)__pyx_t_2));
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":416
 *             keys.encode(), usedforsecurity=False).hexdigest()
 *         mod_name = mod_name.replace('.', '')
 *         mod_filename = mod_name + _ext_suffix             # <<<<<<<<<<<<<<
 * 
 *         # Check if the module is already cached on disk. If not, we compile.
*/
  __pyx_t_2 = __Pyx_PyUnicode_ConcatSafe(__pyx_v_mod_name, __pyx_v_4cupy_3fft_9_callback__ext_suffix); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 416, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_v_mod_filename = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":419
 * 
 *         # Check if the module is already cached on disk. If not, we compile.
 *         cache_dir = get_cache_dir()             # <<<<<<<<<<<<<<
 *         if not os.path.isdir(cache_dir):
 *             os.makedirs(cache_dir, exist_ok=True)
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_get_cache_dir); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 419, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_9 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_3))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_3, __pyx__function);
    __pyx_t_9 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL};
    __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 419, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 419, __pyx_L1_error)
  __pyx_v_cache_dir = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":420
 *         # Check if the module is already cached on disk. If not, we compile.
 *         cache_dir = get_cache_dir()
 *         if not os.path.isdir(cache_dir):             # <<<<<<<<<<<<<<
 *             os.makedirs(cache_dir, exist_ok=True)
 *         path = os.path.join(cache_dir, mod_filename)
*/
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 420, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 420, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  __pyx_t_3 = __pyx_t_7;
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_cache_dir};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_isdir, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 420, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 420, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  __pyx_t_10 = (!__pyx_t_1);
  if (__pyx_t_10) {

    /* "cupy/fft/_callback.pyx":421
 *         cache_dir = get_cache_dir()
 *         if not os.path.isdir(cache_dir):
 *             os.makedirs(cache_dir, exist_ok=True)             # <<<<<<<<<<<<<<
 *         path = os.path.join(cache_dir, mod_filename)
 *         if not os.path.isfile(path):
*/
    __pyx_t_7 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 421, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_makedirs); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 421, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    __pyx_t_9 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_4))) {
      __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_4);
      assert(__pyx_t_7);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
      __Pyx_INCREF(__pyx_t_7);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
      __pyx_t_9 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_7, __pyx_v_cache_dir};
      __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 421, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
      if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_exist_ok, Py_True, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 421, __pyx_L1_error)
      __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3);
      __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 421, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":420
 *         # Check if the module is already cached on disk. If not, we compile.
 *         cache_dir = get_cache_dir()
 *         if not os.path.isdir(cache_dir):             # <<<<<<<<<<<<<<
 *             os.makedirs(cache_dir, exist_ok=True)
 *         path = os.path.join(cache_dir, mod_filename)
*/
  }

  /* "cupy/fft/_callback.pyx":422
 *         if not os.path.isdir(cache_dir):
 *             os.makedirs(cache_dir, exist_ok=True)
 *         path = os.path.join(cache_dir, mod_filename)             # <<<<<<<<<<<<<<
 *         if not os.path.isfile(path):
 *             # Set up temp directory; it must be under the cache directory so
*/
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 422, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 422, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
  __pyx_t_4 = __pyx_t_7;
  __Pyx_INCREF(__pyx_t_4);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_cache_dir, __pyx_v_mod_filename};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 422, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 422, __pyx_L1_error)
  __pyx_v_path = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":423
 *             os.makedirs(cache_dir, exist_ok=True)
 *         path = os.path.join(cache_dir, mod_filename)
 *         if not os.path.isfile(path):             # <<<<<<<<<<<<<<
 *             # Set up temp directory; it must be under the cache directory so
 *             # that atomic moves within the same filesystem can be guaranteed
*/
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 423, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 423, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  __pyx_t_7 = __pyx_t_3;
  __Pyx_INCREF(__pyx_t_7);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_v_path};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_isfile, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 423, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_10 < 0))) __PYX_ERR(0, 423, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  __pyx_t_1 = (!__pyx_t_10);
  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":426
 *             # Set up temp directory; it must be under the cache directory so
 *             # that atomic moves within the same filesystem can be guaranteed
 *             tempdir_obj = tempfile.TemporaryDirectory(dir=cache_dir)             # <<<<<<<<<<<<<<
 *             tempdir = tempdir_obj.name
 * 
*/
    __pyx_t_3 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_tempfile); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 426, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_TemporaryDirectory); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 426, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __pyx_t_9 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_4))) {
      __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
      assert(__pyx_t_3);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
      __Pyx_INCREF(__pyx_t_3);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
      __pyx_t_9 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, NULL};
      __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 426, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_7);
      if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dir, __pyx_v_cache_dir, __pyx_t_7, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 426, __pyx_L1_error)
      __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7);
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 426, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    __pyx_v_tempdir_obj = __pyx_t_2;
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":427
 *             # that atomic moves within the same filesystem can be guaranteed
 *             tempdir_obj = tempfile.TemporaryDirectory(dir=cache_dir)
 *             tempdir = tempdir_obj.name             # <<<<<<<<<<<<<<
 * 
 *             # Cythonize the Cython code to produce a c++ source file
*/
    __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_tempdir_obj, __pyx_mstate_global->__pyx_n_u_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 427, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 427, __pyx_L1_error)
    __pyx_v_tempdir = ((PyObject*)__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":430
 * 
 *             # Cythonize the Cython code to produce a c++ source file
 *             _cythonize(tempdir, mod_name)             # <<<<<<<<<<<<<<
 * 
 *             # Compile the Python module
*/
    __pyx_f_4cupy_3fft_9_callback__cythonize(__pyx_v_tempdir, __pyx_v_mod_name); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 430, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":433
 * 
 *             # Compile the Python module
 *             obj_host = os.path.join(tempdir, mod_name + '.o')             # <<<<<<<<<<<<<<
 *             _mod_compile(tempdir, mod_name, obj_host)
 * 
*/
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 433, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 433, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __pyx_t_4 = __pyx_t_3;
    __Pyx_INCREF(__pyx_t_4);
    __pyx_t_7 = __Pyx_PyUnicode_Concat(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_o_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 433, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_tempdir, __pyx_t_7};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 433, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 433, __pyx_L1_error)
    __pyx_v_obj_host = ((PyObject*)__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":434
 *             # Compile the Python module
 *             obj_host = os.path.join(tempdir, mod_name + '.o')
 *             _mod_compile(tempdir, mod_name, obj_host)             # <<<<<<<<<<<<<<
 * 
 *             # Prune libcufft_static.a for the target arch and cache it
*/
    __pyx_f_4cupy_3fft_9_callback__mod_compile(__pyx_v_tempdir, __pyx_v_mod_name, __pyx_v_obj_host); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 434, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":438
 *             # Prune libcufft_static.a for the target arch and cache it
 *             cufft_lib_pruned = _prune(
 *                 tempdir, cache_dir, str(_cufft_ver), arch)             # <<<<<<<<<<<<<<
 * 
 *             # Dump and compile device code using nvcc
*/
    __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_4cupy_3fft_9_callback__cufft_ver); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 438, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_3 = __Pyx_PyObject_Unicode(__pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 438, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":437
 * 
 *             # Prune libcufft_static.a for the target arch and cache it
 *             cufft_lib_pruned = _prune(             # <<<<<<<<<<<<<<
 *                 tempdir, cache_dir, str(_cufft_ver), arch)
 * 
*/
    __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback__prune(__pyx_v_tempdir, __pyx_v_cache_dir, ((PyObject*)__pyx_t_3), __pyx_v_arch); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 437, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    __pyx_v_cufft_lib_pruned = ((PyObject*)__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":441
 * 
 *             # Dump and compile device code using nvcc
 *             obj_dev = os.path.join(tempdir, mod_name + '_dev.o')             # <<<<<<<<<<<<<<
 *             _nvcc_compile(tempdir, mod_name, cb_load, cb_store, obj_dev, arch)
 * 
*/
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 441, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_path); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    __pyx_t_3 = __pyx_t_4;
    __Pyx_INCREF(__pyx_t_3);
    __pyx_t_7 = __Pyx_PyUnicode_Concat(__pyx_v_mod_name, __pyx_mstate_global->__pyx_kp_u_dev_o); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 441, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_v_tempdir, __pyx_t_7};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_join, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 441, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 441, __pyx_L1_error)
    __pyx_v_obj_dev = ((PyObject*)__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":442
 *             # Dump and compile device code using nvcc
 *             obj_dev = os.path.join(tempdir, mod_name + '_dev.o')
 *             _nvcc_compile(tempdir, mod_name, cb_load, cb_store, obj_dev, arch)             # <<<<<<<<<<<<<<
 * 
 *             # Use nvcc to link and generate a shared library, and place it in
*/
    __pyx_f_4cupy_3fft_9_callback__nvcc_compile(__pyx_v_tempdir, __pyx_v_mod_name, __pyx_v_cb_load, __pyx_v_cb_store, __pyx_v_obj_dev, __pyx_v_arch); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 442, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":446
 *             # Use nvcc to link and generate a shared library, and place it in
 *             # the disk cache
 *             _nvcc_link(tempdir, obj_host, obj_dev, arch, mod_filename,             # <<<<<<<<<<<<<<
 *                        cache_dir, cufft_lib_pruned)
 * 
*/
    __pyx_f_4cupy_3fft_9_callback__nvcc_link(__pyx_v_tempdir, __pyx_v_obj_host, __pyx_v_obj_dev, __pyx_v_arch, __pyx_v_mod_filename, __pyx_v_cache_dir, __pyx_v_cufft_lib_pruned); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 446, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":450
 * 
 *             # Clean up build directory
 *             tempdir_obj.cleanup()             # <<<<<<<<<<<<<<
 * 
 *         # Load the Python module
*/
    __pyx_t_4 = __pyx_v_tempdir_obj;
    __Pyx_INCREF(__pyx_t_4);
    __pyx_t_9 = 0;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL};
      __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_cleanup, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 450, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":423
 *             os.makedirs(cache_dir, exist_ok=True)
 *         path = os.path.join(cache_dir, mod_filename)
 *         if not os.path.isfile(path):             # <<<<<<<<<<<<<<
 *             # Set up temp directory; it must be under the cache directory so
 *             # that atomic moves within the same filesystem can be guaranteed
*/
  }

  /* "cupy/fft/_callback.pyx":453
 * 
 *         # Load the Python module
 *         spec = importlib.util.spec_from_file_location(mod_name, path)             # <<<<<<<<<<<<<<
 *         self.mod = module = importlib.util.module_from_spec(spec)
 *         sys.modules[mod_name] = module
*/
  __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_importlib); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 453, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_util); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 453, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
  __pyx_t_4 = __pyx_t_3;
  __Pyx_INCREF(__pyx_t_4);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_v_mod_name, __pyx_v_path};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_spec_from_file_location, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 453, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_v_spec = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":454
 *         # Load the Python module
 *         spec = importlib.util.spec_from_file_location(mod_name, path)
 *         self.mod = module = importlib.util.module_from_spec(spec)             # <<<<<<<<<<<<<<
 *         sys.modules[mod_name] = module
 *         spec.loader.exec_module(module)
*/
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_importlib); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 454, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_util); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 454, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  __pyx_t_3 = __pyx_t_7;
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_spec};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_module_from_spec, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 454, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __Pyx_INCREF(__pyx_t_2);
  __Pyx_GIVEREF(__pyx_t_2);
  __Pyx_GOTREF(__pyx_v_self->mod);
  __Pyx_DECREF(__pyx_v_self->mod);
  __pyx_v_self->mod = __pyx_t_2;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_v_module = __pyx_t_2;
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":455
 *         spec = importlib.util.spec_from_file_location(mod_name, path)
 *         self.mod = module = importlib.util.module_from_spec(spec)
 *         sys.modules[mod_name] = module             # <<<<<<<<<<<<<<
 *         spec.loader.exec_module(module)
 * 
*/
  __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_sys); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 455, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_modules); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 455, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_7);
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  if (unlikely((PyObject_SetItem(__pyx_t_7, __pyx_v_mod_name, __pyx_v_module) < 0))) __PYX_ERR(0, 455, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

  /* "cupy/fft/_callback.pyx":456
 *         self.mod = module = importlib.util.module_from_spec(spec)
 *         sys.modules[mod_name] = module
 *         spec.loader.exec_module(module)             # <<<<<<<<<<<<<<
 * 
 *     cpdef create_plan(self, tuple plan_info):
*/
  __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_spec, __pyx_mstate_global->__pyx_n_u_loader); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 456, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_2 = __pyx_t_3;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_9 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_module};
    __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_exec_module, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 456, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
  }
  __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

  /* "cupy/fft/_callback.pyx":376
 *         object mod
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  str cb_load='',
 *                  str cb_store='',
*/

  /* function exit code */
  __pyx_r = 0;
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = -1;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_arch);
  __Pyx_XDECREF(__pyx_v_tempdir);
  __Pyx_XDECREF(__pyx_v_obj_host);
  __Pyx_XDECREF(__pyx_v_obj_dev);
  __Pyx_XDECREF(__pyx_v_mod_name);
  __Pyx_XDECREF(__pyx_v_mod_filename);
  __Pyx_XDECREF(__pyx_v_cache_dir);
  __Pyx_XDECREF(__pyx_v_path);
  __Pyx_XDECREF(__pyx_v_cufft_lib_pruned);
  __Pyx_XDECREF(__pyx_v_keys);
  __Pyx_XDECREF(__pyx_v_tempdir_obj);
  __Pyx_XDECREF(__pyx_v_spec);
  __Pyx_XDECREF(__pyx_v_module);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":458
 *         spec.loader.exec_module(module)
 * 
 *     cpdef create_plan(self, tuple plan_info):             # <<<<<<<<<<<<<<
 *         cdef str plan_type
 *         cdef tuple plan_args
*/

static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_3create_plan(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_create_plan(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan_info, int __pyx_skip_dispatch) {
  PyObject *__pyx_v_plan_type = 0;
  PyObject *__pyx_v_plan_args = 0;
  PyObject *__pyx_v_plan = NULL;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("create_plan", 0);
  /* Check if called by wrapper */
  if (unlikely(__pyx_skip_dispatch)) ;
  /* Check if overridden in Python */
  else if (
  #if !CYTHON_USE_TYPE_SLOTS
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self)) != __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager &&
  __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), Py_TPFLAGS_HAVE_GC))
  #else
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0 || __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))
  #endif
  ) {
    #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
    if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) {
      PY_UINT64_T __pyx_typedict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      #endif
      __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_create_plan); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 458, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
      if (!__Pyx_IsSameCFunction(__pyx_t_1, (void(*)(void)) __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_3create_plan)) {
        __Pyx_XDECREF(__pyx_r);
        __pyx_t_3 = NULL;
        __Pyx_INCREF(__pyx_t_1);
        __pyx_t_4 = __pyx_t_1; 
        __pyx_t_5 = 1;
        #if CYTHON_UNPACK_METHODS
        if (unlikely(PyMethod_Check(__pyx_t_4))) {
          __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
          assert(__pyx_t_3);
          PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
          __Pyx_INCREF(__pyx_t_3);
          __Pyx_INCREF(__pyx__function);
          __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
          __pyx_t_5 = 0;
        }
        #endif
        {
          PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_plan_info};
          __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
          if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 458, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_2);
        }
        __pyx_r = __pyx_t_2;
        __pyx_t_2 = 0;
        __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
        goto __pyx_L0;
      }
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
      __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self));
      if (unlikely(__pyx_typedict_guard != __pyx_tp_dict_version)) {
        __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
      }
      #endif
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    }
    #endif
  }

  /* "cupy/fft/_callback.pyx":462
 *         cdef tuple plan_args
 * 
 *         plan_type, plan_args = plan_info             # <<<<<<<<<<<<<<
 *         plan = getattr(self.mod, plan_type)(*plan_args)
 *         return plan
*/
  if (likely(__pyx_v_plan_info != Py_None)) {
    PyObject* sequence = __pyx_v_plan_info;
    Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence);
    if (unlikely(size != 2)) {
      if (size > 2) __Pyx_RaiseTooManyValuesError(2);
      else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
      __PYX_ERR(0, 462, __pyx_L1_error)
    }
    #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
    __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0);
    __Pyx_INCREF(__pyx_t_1);
    __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1);
    __Pyx_INCREF(__pyx_t_2);
    #else
    __pyx_t_1 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 462, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 462, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    #endif
  } else {
    __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(0, 462, __pyx_L1_error)
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 462, __pyx_L1_error)
  if (!(likely(PyTuple_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_2))) __PYX_ERR(0, 462, __pyx_L1_error)
  __pyx_v_plan_type = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_v_plan_args = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":463
 * 
 *         plan_type, plan_args = plan_info
 *         plan = getattr(self.mod, plan_type)(*plan_args)             # <<<<<<<<<<<<<<
 *         return plan
 * 
*/
  __pyx_t_2 = __pyx_v_self->mod;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_1 = __Pyx_GetAttr(__pyx_t_2, __pyx_v_plan_type); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 463, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  if (unlikely(__pyx_v_plan_args == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
    __PYX_ERR(0, 463, __pyx_L1_error)
  }
  __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_v_plan_args, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 463, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_v_plan = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":464
 *         plan_type, plan_args = plan_info
 *         plan = getattr(self.mod, plan_type)(*plan_args)
 *         return plan             # <<<<<<<<<<<<<<
 * 
 *     cpdef set_callbacks(self, plan):
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_plan);
  __pyx_r = __pyx_v_plan;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":458
 *         spec.loader.exec_module(module)
 * 
 *     cpdef create_plan(self, tuple plan_info):             # <<<<<<<<<<<<<<
 *         cdef str plan_type
 *         cdef tuple plan_args
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.create_plan", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_plan_type);
  __Pyx_XDECREF(__pyx_v_plan_args);
  __Pyx_XDECREF(__pyx_v_plan);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_3create_plan(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_2create_plan, "_LegacyCallbackManager.create_plan(self, tuple plan_info)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_3create_plan(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v_plan_info = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("create_plan (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_plan_info,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 458, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 458, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "create_plan", 0) < (0)) __PYX_ERR(0, 458, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("create_plan", 1, 1, 1, i); __PYX_ERR(0, 458, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 458, __pyx_L3_error)
    }
    __pyx_v_plan_info = ((PyObject*)values[0]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("create_plan", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 458, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.create_plan", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_plan_info), (&PyTuple_Type), 1, "plan_info", 1))) __PYX_ERR(0, 458, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_2create_plan(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self), __pyx_v_plan_info);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_2create_plan(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan_info) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("create_plan", 0);
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = __pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_create_plan(__pyx_v_self, __pyx_v_plan_info, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 458, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_r = __pyx_t_1;
  __pyx_t_1 = 0;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.create_plan", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":466
 *         return plan
 * 
 *     cpdef set_callbacks(self, plan):             # <<<<<<<<<<<<<<
 *         '''Set the load/store callbacks by making calls to
 *         ``cufftXtSetCallback``.
*/

static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_5set_callbacks(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan, int __pyx_skip_dispatch) {
  PyObject *__pyx_v_cufft = NULL;
  PyObject *__pyx_v_CUFFT_C2C = NULL;
  PyObject *__pyx_v_CUFFT_C2R = NULL;
  PyObject *__pyx_v_CUFFT_R2C = NULL;
  PyObject *__pyx_v_CUFFT_Z2Z = NULL;
  PyObject *__pyx_v_CUFFT_Z2D = NULL;
  PyObject *__pyx_v_CUFFT_D2Z = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_COMPLEX = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_REAL = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_REAL_DOUBLE = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_COMPLEX = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_REAL = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_REAL_DOUBLE = NULL;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data = 0;
  intptr_t __pyx_v_cb_load_ptr;
  intptr_t __pyx_v_cb_store_ptr;
  PyObject *__pyx_v_fft_type = NULL;
  PyObject *__pyx_v_cb_load_type = NULL;
  PyObject *__pyx_v_cb_store_type = NULL;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  int __pyx_t_6;
  intptr_t __pyx_t_7;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("set_callbacks", 0);
  /* Check if called by wrapper */
  if (unlikely(__pyx_skip_dispatch)) ;
  /* Check if overridden in Python */
  else if (
  #if !CYTHON_USE_TYPE_SLOTS
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self)) != __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager &&
  __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), Py_TPFLAGS_HAVE_GC))
  #else
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0 || __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))
  #endif
  ) {
    #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
    if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) {
      PY_UINT64_T __pyx_typedict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      #endif
      __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_set_callbacks); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 466, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
      if (!__Pyx_IsSameCFunction(__pyx_t_1, (void(*)(void)) __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_5set_callbacks)) {
        __Pyx_XDECREF(__pyx_r);
        __pyx_t_3 = NULL;
        __Pyx_INCREF(__pyx_t_1);
        __pyx_t_4 = __pyx_t_1; 
        __pyx_t_5 = 1;
        #if CYTHON_UNPACK_METHODS
        if (unlikely(PyMethod_Check(__pyx_t_4))) {
          __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
          assert(__pyx_t_3);
          PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
          __Pyx_INCREF(__pyx_t_3);
          __Pyx_INCREF(__pyx__function);
          __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
          __pyx_t_5 = 0;
        }
        #endif
        {
          PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_plan};
          __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
          if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 466, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_2);
        }
        __pyx_r = __pyx_t_2;
        __pyx_t_2 = 0;
        __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
        goto __pyx_L0;
      }
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
      __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self));
      if (unlikely(__pyx_typedict_guard != __pyx_tp_dict_version)) {
        __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
      }
      #endif
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    }
    #endif
  }

  /* "cupy/fft/_callback.pyx":482
 *         # Note: Do NOT use any object/function from the global cupy.cuda.cufft
 *         # module!
 *         cufft = self.mod             # <<<<<<<<<<<<<<
 *         CUFFT_C2C = cufft.CUFFT_C2C
 *         CUFFT_C2R = cufft.CUFFT_C2R
*/
  __pyx_t_1 = __pyx_v_self->mod;
  __Pyx_INCREF(__pyx_t_1);
  __pyx_v_cufft = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":483
 *         # module!
 *         cufft = self.mod
 *         CUFFT_C2C = cufft.CUFFT_C2C             # <<<<<<<<<<<<<<
 *         CUFFT_C2R = cufft.CUFFT_C2R
 *         CUFFT_R2C = cufft.CUFFT_R2C
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_C2C); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 483, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_C2C = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":484
 *         cufft = self.mod
 *         CUFFT_C2C = cufft.CUFFT_C2C
 *         CUFFT_C2R = cufft.CUFFT_C2R             # <<<<<<<<<<<<<<
 *         CUFFT_R2C = cufft.CUFFT_R2C
 *         CUFFT_Z2Z = cufft.CUFFT_Z2Z
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_C2R); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 484, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_C2R = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":485
 *         CUFFT_C2C = cufft.CUFFT_C2C
 *         CUFFT_C2R = cufft.CUFFT_C2R
 *         CUFFT_R2C = cufft.CUFFT_R2C             # <<<<<<<<<<<<<<
 *         CUFFT_Z2Z = cufft.CUFFT_Z2Z
 *         CUFFT_Z2D = cufft.CUFFT_Z2D
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_R2C); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 485, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_R2C = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":486
 *         CUFFT_C2R = cufft.CUFFT_C2R
 *         CUFFT_R2C = cufft.CUFFT_R2C
 *         CUFFT_Z2Z = cufft.CUFFT_Z2Z             # <<<<<<<<<<<<<<
 *         CUFFT_Z2D = cufft.CUFFT_Z2D
 *         CUFFT_D2Z = cufft.CUFFT_D2Z
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_Z2Z); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 486, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_Z2Z = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":487
 *         CUFFT_R2C = cufft.CUFFT_R2C
 *         CUFFT_Z2Z = cufft.CUFFT_Z2Z
 *         CUFFT_Z2D = cufft.CUFFT_Z2D             # <<<<<<<<<<<<<<
 *         CUFFT_D2Z = cufft.CUFFT_D2Z
 *         CUFFT_CB_LD_COMPLEX = cufft.CUFFT_CB_LD_COMPLEX
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_Z2D); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_Z2D = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":488
 *         CUFFT_Z2Z = cufft.CUFFT_Z2Z
 *         CUFFT_Z2D = cufft.CUFFT_Z2D
 *         CUFFT_D2Z = cufft.CUFFT_D2Z             # <<<<<<<<<<<<<<
 *         CUFFT_CB_LD_COMPLEX = cufft.CUFFT_CB_LD_COMPLEX
 *         CUFFT_CB_LD_COMPLEX_DOUBLE = cufft.CUFFT_CB_LD_COMPLEX_DOUBLE
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_D2Z); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 488, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_D2Z = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":489
 *         CUFFT_Z2D = cufft.CUFFT_Z2D
 *         CUFFT_D2Z = cufft.CUFFT_D2Z
 *         CUFFT_CB_LD_COMPLEX = cufft.CUFFT_CB_LD_COMPLEX             # <<<<<<<<<<<<<<
 *         CUFFT_CB_LD_COMPLEX_DOUBLE = cufft.CUFFT_CB_LD_COMPLEX_DOUBLE
 *         CUFFT_CB_LD_REAL = cufft.CUFFT_CB_LD_REAL
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_COMPLEX); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 489, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_LD_COMPLEX = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":490
 *         CUFFT_D2Z = cufft.CUFFT_D2Z
 *         CUFFT_CB_LD_COMPLEX = cufft.CUFFT_CB_LD_COMPLEX
 *         CUFFT_CB_LD_COMPLEX_DOUBLE = cufft.CUFFT_CB_LD_COMPLEX_DOUBLE             # <<<<<<<<<<<<<<
 *         CUFFT_CB_LD_REAL = cufft.CUFFT_CB_LD_REAL
 *         CUFFT_CB_LD_REAL_DOUBLE = cufft.CUFFT_CB_LD_REAL_DOUBLE
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_COMPLEX_DOUBLE); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 490, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":491
 *         CUFFT_CB_LD_COMPLEX = cufft.CUFFT_CB_LD_COMPLEX
 *         CUFFT_CB_LD_COMPLEX_DOUBLE = cufft.CUFFT_CB_LD_COMPLEX_DOUBLE
 *         CUFFT_CB_LD_REAL = cufft.CUFFT_CB_LD_REAL             # <<<<<<<<<<<<<<
 *         CUFFT_CB_LD_REAL_DOUBLE = cufft.CUFFT_CB_LD_REAL_DOUBLE
 *         CUFFT_CB_ST_COMPLEX = cufft.CUFFT_CB_ST_COMPLEX
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_REAL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 491, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_LD_REAL = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":492
 *         CUFFT_CB_LD_COMPLEX_DOUBLE = cufft.CUFFT_CB_LD_COMPLEX_DOUBLE
 *         CUFFT_CB_LD_REAL = cufft.CUFFT_CB_LD_REAL
 *         CUFFT_CB_LD_REAL_DOUBLE = cufft.CUFFT_CB_LD_REAL_DOUBLE             # <<<<<<<<<<<<<<
 *         CUFFT_CB_ST_COMPLEX = cufft.CUFFT_CB_ST_COMPLEX
 *         CUFFT_CB_ST_COMPLEX_DOUBLE = cufft.CUFFT_CB_ST_COMPLEX_DOUBLE
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_REAL_DOUBLE); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 492, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_LD_REAL_DOUBLE = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":493
 *         CUFFT_CB_LD_REAL = cufft.CUFFT_CB_LD_REAL
 *         CUFFT_CB_LD_REAL_DOUBLE = cufft.CUFFT_CB_LD_REAL_DOUBLE
 *         CUFFT_CB_ST_COMPLEX = cufft.CUFFT_CB_ST_COMPLEX             # <<<<<<<<<<<<<<
 *         CUFFT_CB_ST_COMPLEX_DOUBLE = cufft.CUFFT_CB_ST_COMPLEX_DOUBLE
 *         CUFFT_CB_ST_REAL = cufft.CUFFT_CB_ST_REAL
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_COMPLEX); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 493, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_ST_COMPLEX = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":494
 *         CUFFT_CB_LD_REAL_DOUBLE = cufft.CUFFT_CB_LD_REAL_DOUBLE
 *         CUFFT_CB_ST_COMPLEX = cufft.CUFFT_CB_ST_COMPLEX
 *         CUFFT_CB_ST_COMPLEX_DOUBLE = cufft.CUFFT_CB_ST_COMPLEX_DOUBLE             # <<<<<<<<<<<<<<
 *         CUFFT_CB_ST_REAL = cufft.CUFFT_CB_ST_REAL
 *         CUFFT_CB_ST_REAL_DOUBLE = cufft.CUFFT_CB_ST_REAL_DOUBLE
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_COMPLEX_DOUBLE); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":495
 *         CUFFT_CB_ST_COMPLEX = cufft.CUFFT_CB_ST_COMPLEX
 *         CUFFT_CB_ST_COMPLEX_DOUBLE = cufft.CUFFT_CB_ST_COMPLEX_DOUBLE
 *         CUFFT_CB_ST_REAL = cufft.CUFFT_CB_ST_REAL             # <<<<<<<<<<<<<<
 *         CUFFT_CB_ST_REAL_DOUBLE = cufft.CUFFT_CB_ST_REAL_DOUBLE
 * 
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_REAL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 495, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_ST_REAL = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":496
 *         CUFFT_CB_ST_COMPLEX_DOUBLE = cufft.CUFFT_CB_ST_COMPLEX_DOUBLE
 *         CUFFT_CB_ST_REAL = cufft.CUFFT_CB_ST_REAL
 *         CUFFT_CB_ST_REAL_DOUBLE = cufft.CUFFT_CB_ST_REAL_DOUBLE             # <<<<<<<<<<<<<<
 * 
 *         cdef MemoryPointer cb_load_data = self.cb_load_data
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cufft, __pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_REAL_DOUBLE); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 496, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_CUFFT_CB_ST_REAL_DOUBLE = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":498
 *         CUFFT_CB_ST_REAL_DOUBLE = cufft.CUFFT_CB_ST_REAL_DOUBLE
 * 
 *         cdef MemoryPointer cb_load_data = self.cb_load_data             # <<<<<<<<<<<<<<
 *         cdef MemoryPointer cb_store_data = self.cb_store_data
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __Pyx_INCREF(__pyx_t_1);
  __pyx_v_cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":499
 * 
 *         cdef MemoryPointer cb_load_data = self.cb_load_data
 *         cdef MemoryPointer cb_store_data = self.cb_store_data             # <<<<<<<<<<<<<<
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
 * 
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __Pyx_INCREF(__pyx_t_1);
  __pyx_v_cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":500
 *         cdef MemoryPointer cb_load_data = self.cb_load_data
 *         cdef MemoryPointer cb_store_data = self.cb_store_data
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0             # <<<<<<<<<<<<<<
 * 
 *         fft_type = plan.fft_type
*/
  __pyx_v_cb_load_ptr = 0;
  __pyx_v_cb_store_ptr = 0;

  /* "cupy/fft/_callback.pyx":502
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
 * 
 *         fft_type = plan.fft_type             # <<<<<<<<<<<<<<
 *         if fft_type == CUFFT_C2C:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
*/
  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_plan, __pyx_mstate_global->__pyx_n_u_fft_type); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 502, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_fft_type = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":503
 * 
 *         fft_type = plan.fft_type
 *         if fft_type == CUFFT_C2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_C2C, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 503, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 503, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":504
 *         fft_type = plan.fft_type
 *         if fft_type == CUFFT_C2C:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:
*/
    if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 504, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":505
 *         if fft_type == CUFFT_C2C:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_R2C:
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
*/
    if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 505, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":503
 * 
 *         fft_type = plan.fft_type
 *         if fft_type == CUFFT_C2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":506
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_R2C, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 506, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 506, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":507
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:
*/
    if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 507, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_REAL);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_REAL;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":508
 *         elif fft_type == CUFFT_R2C:
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_C2R:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
*/
    if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 508, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":506
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":509
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_C2R, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 509, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 509, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":510
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:
*/
    if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 510, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":511
 *         elif fft_type == CUFFT_C2R:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_Z2Z:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
*/
    if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 511, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_REAL);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_REAL;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":509
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":512
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_Z2Z, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 512, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 512, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":513
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:
*/
    if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 513, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":514
 *         elif fft_type == CUFFT_Z2Z:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_D2Z:
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
*/
    if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 514, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":512
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":515
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_D2Z, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 515, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 515, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":516
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:
*/
    if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 516, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_REAL_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_REAL_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":517
 *         elif fft_type == CUFFT_D2Z:
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_Z2D:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
*/
    if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 517, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":515
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":518
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_Z2D, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 518, __pyx_L1_error)
  __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 518, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (likely(__pyx_t_6)) {

    /* "cupy/fft/_callback.pyx":519
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
 *         else:
*/
    if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 519, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":520
 *         elif fft_type == CUFFT_Z2D:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         else:
 *             raise ValueError
*/
    if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
    else
    {
      Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
      if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 520, __pyx_L1_error)
      __pyx_t_6 = (__pyx_temp != 0);
    }

    if (__pyx_t_6) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_REAL_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_REAL_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":518
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":522
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
 *         else:
 *             raise ValueError             # <<<<<<<<<<<<<<
 * 
 *         if self.cb_load:
*/
  /*else*/ {
    __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), 0, 0, 0);
    __PYX_ERR(0, 522, __pyx_L1_error)
  }
  __pyx_L3:;

  /* "cupy/fft/_callback.pyx":524
 *             raise ValueError
 * 
 *         if self.cb_load:             # <<<<<<<<<<<<<<
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr
*/
  if (__pyx_v_self->cb_load == Py_None) __pyx_t_6 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_load);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 524, __pyx_L1_error)
    __pyx_t_6 = (__pyx_temp != 0);
  }

  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":525
 * 
 *         if self.cb_load:
 *             if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setCallback(
*/
    __pyx_t_6 = (((PyObject *)__pyx_v_cb_load_data) != Py_None);
    if (__pyx_t_6) {

      /* "cupy/fft/_callback.pyx":526
 *         if self.cb_load:
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr             # <<<<<<<<<<<<<<
 *             cufft.setCallback(
 *                 plan.handle, cb_load_type, True, cb_load_ptr)
*/
      __pyx_t_7 = __pyx_v_cb_load_data->ptr;
      __pyx_v_cb_load_ptr = __pyx_t_7;

      /* "cupy/fft/_callback.pyx":525
 * 
 *         if self.cb_load:
 *             if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setCallback(
*/
    }

    /* "cupy/fft/_callback.pyx":527
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setCallback(             # <<<<<<<<<<<<<<
 *                 plan.handle, cb_load_type, True, cb_load_ptr)
 *         if self.cb_store:
*/
    __pyx_t_2 = __pyx_v_cufft;
    __Pyx_INCREF(__pyx_t_2);

    /* "cupy/fft/_callback.pyx":528
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setCallback(
 *                 plan.handle, cb_load_type, True, cb_load_ptr)             # <<<<<<<<<<<<<<
 *         if self.cb_store:
 *             if cb_store_data is not None:
*/
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_plan, __pyx_mstate_global->__pyx_n_u_handle); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 528, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_cb_load_ptr); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 528, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __pyx_t_5 = 0;
    {
      PyObject *__pyx_callargs[5] = {__pyx_t_2, __pyx_t_4, __pyx_v_cb_load_type, Py_True, __pyx_t_3};
      __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_setCallback, __pyx_callargs+__pyx_t_5, (5-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 527, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":524
 *             raise ValueError
 * 
 *         if self.cb_load:             # <<<<<<<<<<<<<<
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr
*/
  }

  /* "cupy/fft/_callback.pyx":529
 *             cufft.setCallback(
 *                 plan.handle, cb_load_type, True, cb_load_ptr)
 *         if self.cb_store:             # <<<<<<<<<<<<<<
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr
*/
  if (__pyx_v_self->cb_store == Py_None) __pyx_t_6 = 0;
  else
  {
    Py_ssize_t __pyx_temp = __Pyx_PyUnicode_IS_TRUE(__pyx_v_self->cb_store);
    if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 529, __pyx_L1_error)
    __pyx_t_6 = (__pyx_temp != 0);
  }

  if (__pyx_t_6) {

    /* "cupy/fft/_callback.pyx":530
 *                 plan.handle, cb_load_type, True, cb_load_ptr)
 *         if self.cb_store:
 *             if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setCallback(
*/
    __pyx_t_6 = (((PyObject *)__pyx_v_cb_store_data) != Py_None);
    if (__pyx_t_6) {

      /* "cupy/fft/_callback.pyx":531
 *         if self.cb_store:
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr             # <<<<<<<<<<<<<<
 *             cufft.setCallback(
 *                 plan.handle, cb_store_type, False, cb_store_ptr)
*/
      __pyx_t_7 = __pyx_v_cb_store_data->ptr;
      __pyx_v_cb_store_ptr = __pyx_t_7;

      /* "cupy/fft/_callback.pyx":530
 *                 plan.handle, cb_load_type, True, cb_load_ptr)
 *         if self.cb_store:
 *             if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setCallback(
*/
    }

    /* "cupy/fft/_callback.pyx":532
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setCallback(             # <<<<<<<<<<<<<<
 *                 plan.handle, cb_store_type, False, cb_store_ptr)
 * 
*/
    __pyx_t_3 = __pyx_v_cufft;
    __Pyx_INCREF(__pyx_t_3);

    /* "cupy/fft/_callback.pyx":533
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setCallback(
 *                 plan.handle, cb_store_type, False, cb_store_ptr)             # <<<<<<<<<<<<<<
 * 
 * 
*/
    __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_plan, __pyx_mstate_global->__pyx_n_u_handle); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 533, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_cb_store_ptr); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 533, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __pyx_t_5 = 0;
    {
      PyObject *__pyx_callargs[5] = {__pyx_t_3, __pyx_t_4, __pyx_v_cb_store_type, Py_False, __pyx_t_2};
      __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_setCallback, __pyx_callargs+__pyx_t_5, (5-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 532, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":529
 *             cufft.setCallback(
 *                 plan.handle, cb_load_type, True, cb_load_ptr)
 *         if self.cb_store:             # <<<<<<<<<<<<<<
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr
*/
  }

  /* "cupy/fft/_callback.pyx":466
 *         return plan
 * 
 *     cpdef set_callbacks(self, plan):             # <<<<<<<<<<<<<<
 *         '''Set the load/store callbacks by making calls to
 *         ``cufftXtSetCallback``.
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.set_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_cufft);
  __Pyx_XDECREF(__pyx_v_CUFFT_C2C);
  __Pyx_XDECREF(__pyx_v_CUFFT_C2R);
  __Pyx_XDECREF(__pyx_v_CUFFT_R2C);
  __Pyx_XDECREF(__pyx_v_CUFFT_Z2Z);
  __Pyx_XDECREF(__pyx_v_CUFFT_Z2D);
  __Pyx_XDECREF(__pyx_v_CUFFT_D2Z);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_COMPLEX);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_REAL);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_REAL_DOUBLE);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_COMPLEX);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_REAL);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_REAL_DOUBLE);
  __Pyx_XDECREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_XDECREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_XDECREF(__pyx_v_fft_type);
  __Pyx_XDECREF(__pyx_v_cb_load_type);
  __Pyx_XDECREF(__pyx_v_cb_store_type);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_5set_callbacks(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_4set_callbacks, "_LegacyCallbackManager.set_callbacks(self, plan)\n\nSet the load/store callbacks by making calls to\n``cufftXtSetCallback``.\n\nArgs:\n    plan (:class:`~cupy.cuda.cufft.Plan1d` or\n        :class:`~cupy.cuda.cufft.PlanNd`, optional): A cuFFT plan\n        against which the load/store callbacks are set.\n\n.. note::\n    If :meth:`set_caller_infos` is called, a call to this method must\n    follow.");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_5set_callbacks(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v_plan = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("set_callbacks (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_plan,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 466, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 466, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "set_callbacks", 0) < (0)) __PYX_ERR(0, 466, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("set_callbacks", 1, 1, 1, i); __PYX_ERR(0, 466, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 466, __pyx_L3_error)
    }
    __pyx_v_plan = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("set_callbacks", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 466, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.set_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_4set_callbacks(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self), __pyx_v_plan);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_4set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v_plan) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("set_callbacks", 0);
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = __pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_set_callbacks(__pyx_v_self, __pyx_v_plan, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 466, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_r = __pyx_t_1;
  __pyx_t_1 = 0;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.set_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":372
 * 
 *     cdef:
 *         readonly str cb_load             # <<<<<<<<<<<<<<
 *         readonly str cb_store
 *         object mod
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_7cb_load_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_7cb_load_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_7cb_load___get__(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_7cb_load___get__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_load);
  __pyx_r = __pyx_v_self->cb_load;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":373
 *     cdef:
 *         readonly str cb_load
 *         readonly str cb_store             # <<<<<<<<<<<<<<
 *         object mod
 * 
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_8cb_store_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_8cb_store_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_8cb_store___get__(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_8cb_store___get__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_store);
  __pyx_r = __pyx_v_self->cb_store;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_7__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_6__reduce_cython__, "_LegacyCallbackManager.__reduce_cython__(self)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_7__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return NULL;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_6__reduce_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_6__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self) {
  PyObject *__pyx_v_state = 0;
  PyObject *__pyx_v__dict = 0;
  int __pyx_v_use_setstate;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__reduce_cython__", 0);

  /* "(tree fragment)":5
 *     cdef object _dict
 *     cdef bint use_setstate
 *     state = (self.cb_load, self.cb_load_data, self.cb_store, self.cb_store_data, self.identity, self.mod)             # <<<<<<<<<<<<<<
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
*/
  __pyx_t_1 = PyTuple_New(6); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_INCREF(__pyx_v_self->cb_load);
  __Pyx_GIVEREF(__pyx_v_self->cb_load);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->cb_load) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_self->__pyx_base.cb_load_data)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->cb_store);
  __Pyx_GIVEREF(__pyx_v_self->cb_store);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_self->cb_store) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, ((PyObject *)__pyx_v_self->__pyx_base.cb_store_data)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->__pyx_base.identity);
  __Pyx_GIVEREF(__pyx_v_self->__pyx_base.identity);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 4, __pyx_v_self->__pyx_base.identity) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->mod);
  __Pyx_GIVEREF(__pyx_v_self->mod);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 5, __pyx_v_self->mod) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __pyx_v_state = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":6
 *     cdef bint use_setstate
 *     state = (self.cb_load, self.cb_load_data, self.cb_store, self.cb_store_data, self.identity, self.mod)
 *     _dict = getattr(self, '__dict__', None)             # <<<<<<<<<<<<<<
 *     if _dict is not None and _dict:
 *         state += (_dict,)
*/
  __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 6, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v__dict = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "(tree fragment)":7
 *     state = (self.cb_load, self.cb_load_data, self.cb_store, self.cb_store_data, self.identity, self.mod)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
  __pyx_t_3 = (__pyx_v__dict != Py_None);
  if (__pyx_t_3) {
  } else {
    __pyx_t_2 = __pyx_t_3;
    goto __pyx_L4_bool_binop_done;
  }
  __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(3, 7, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_3;
  __pyx_L4_bool_binop_done:;
  if (__pyx_t_2) {

    /* "(tree fragment)":8
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
 *         state += (_dict,)             # <<<<<<<<<<<<<<
 *         use_setstate = True
 *     else:
*/
    __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(__pyx_v__dict);
    __Pyx_GIVEREF(__pyx_v__dict);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(3, 8, __pyx_L1_error);
    __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "(tree fragment)":9
 *     if _dict is not None and _dict:
 *         state += (_dict,)
 *         use_setstate = True             # <<<<<<<<<<<<<<
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_store is not None or self.cb_store_data is not None or self.identity is not None or self.mod is not None
*/
    __pyx_v_use_setstate = 1;

    /* "(tree fragment)":7
 *     state = (self.cb_load, self.cb_load_data, self.cb_store, self.cb_store_data, self.identity, self.mod)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
    goto __pyx_L3;
  }

  /* "(tree fragment)":11
 *         use_setstate = True
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_store is not None or self.cb_store_data is not None or self.identity is not None or self.mod is not None             # <<<<<<<<<<<<<<
 *     if use_setstate:
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, None), state
*/
  /*else*/ {
    __pyx_t_3 = (__pyx_v_self->cb_load != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (((PyObject *)__pyx_v_self->__pyx_base.cb_load_data) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->cb_store != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (((PyObject *)__pyx_v_self->__pyx_base.cb_store_data) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->__pyx_base.identity != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->mod != Py_None);
    __pyx_t_2 = __pyx_t_3;
    __pyx_L6_bool_binop_done:;
    __pyx_v_use_setstate = __pyx_t_2;
  }
  __pyx_L3:;

  /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_store is not None or self.cb_store_data is not None or self.identity is not None or self.mod is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, None), state
 *     else:
*/
  if (__pyx_v_use_setstate) {

    /* "(tree fragment)":13
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_store is not None or self.cb_store_data is not None or self.identity is not None or self.mod is not None
 *     if use_setstate:
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, None), state             # <<<<<<<<<<<<<<
 *     else:
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, state)
*/
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__LegacyCallbackMa); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_52696597);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_52696597);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_52696597) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(Py_None);
    __Pyx_GIVEREF(Py_None);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_GIVEREF(__pyx_t_4);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_4 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_5;
    __pyx_t_5 = 0;
    goto __pyx_L0;

    /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_store is not None or self.cb_store_data is not None or self.identity is not None or self.mod is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, None), state
 *     else:
*/
  }

  /* "(tree fragment)":15
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, None), state
 *     else:
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, state)             # <<<<<<<<<<<<<<
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__LegacyCallbackManager__set_state(self, __pyx_state)
*/
  /*else*/ {
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__LegacyCallbackMa); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_52696597);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_52696597);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_52696597) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_GIVEREF(__pyx_t_5);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_5 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_4;
    __pyx_t_4 = 0;
    goto __pyx_L0;
  }

  /* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_state);
  __Pyx_XDECREF(__pyx_v__dict);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__LegacyCallbackManager__set_state(self, __pyx_state)
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_9__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_8__setstate_cython__, "_LegacyCallbackManager.__setstate_cython__(self, __pyx_state)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_9__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 16, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(3, 16, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(3, 16, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
    }
    __pyx_v___pyx_state = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(3, 16, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_8__setstate_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v_self), __pyx_v___pyx_state);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_22_LegacyCallbackManager_8__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v_self, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__setstate_cython__", 0);

  /* "(tree fragment)":17
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, state)
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__LegacyCallbackManager__set_state(self, __pyx_state)             # <<<<<<<<<<<<<<
*/
  __pyx_t_1 = __pyx_v___pyx_state;
  __Pyx_INCREF(__pyx_t_1);
  if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(3, 17, __pyx_L1_error)
  if (unlikely(__pyx_t_1 == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
    __PYX_ERR(3, 17, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__LegacyCallbackManager__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 17, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__LegacyCallbackManager, (type(self), 0x3241615, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__LegacyCallbackManager__set_state(self, __pyx_state)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_AddTraceback("cupy.fft._callback._LegacyCallbackManager.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":546
 *         readonly bytes cb_store_lto
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  cb_load=None,
 *                  cb_store=None,
*/

/* Python wrapper */
static int __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static int __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
  PyObject *__pyx_v_cb_load = 0;
  PyObject *__pyx_v_cb_store = 0;
  PyObject *__pyx_v_cb_load_name = 0;
  PyObject *__pyx_v_cb_store_name = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data = 0;
  PyObject *__pyx_v_nvrtc_options = 0;
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[7] = {0,0,0,0,0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1;
  #endif
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_cb_load,&__pyx_mstate_global->__pyx_n_u_cb_store,&__pyx_mstate_global->__pyx_n_u_cb_load_name,&__pyx_mstate_global->__pyx_n_u_cb_store_name,&__pyx_mstate_global->__pyx_n_u_cb_load_data,&__pyx_mstate_global->__pyx_n_u_cb_store_data,&__pyx_mstate_global->__pyx_n_u_nvrtc_options,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 546, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  7:
        values[6] = __Pyx_ArgRef_VARARGS(__pyx_args, 6);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[6])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  6:
        values[5] = __Pyx_ArgRef_VARARGS(__pyx_args, 5);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  5:
        values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  4:
        values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  3:
        values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(0, 546, __pyx_L3_error)

      /* "cupy/fft/_callback.pyx":547
 * 
 *     def __init__(self,
 *                  cb_load=None,             # <<<<<<<<<<<<<<
 *                  cb_store=None,
 *                  str cb_load_name=None,
*/
      if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":548
 *     def __init__(self,
 *                  cb_load=None,
 *                  cb_store=None,             # <<<<<<<<<<<<<<
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
*/
      if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":549
 *                  cb_load=None,
 *                  cb_store=None,
 *                  str cb_load_name=None,             # <<<<<<<<<<<<<<
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
*/
      if (!values[2]) values[2] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":550
 *                  cb_store=None,
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,
*/
      if (!values[3]) values[3] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":551
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_store_data=None,
 *                  tuple nvrtc_options=(),
*/
      if (!values[4]) values[4] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":552
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,             # <<<<<<<<<<<<<<
 *                  tuple nvrtc_options=(),
 *                  ):
*/
      if (!values[5]) values[5] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":553
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,
 *                  tuple nvrtc_options=(),             # <<<<<<<<<<<<<<
 *                  ):
 *         self._sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
*/
      if (!values[6]) values[6] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_empty_tuple));
    } else {
      switch (__pyx_nargs) {
        case  7:
        values[6] = __Pyx_ArgRef_VARARGS(__pyx_args, 6);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[6])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  6:
        values[5] = __Pyx_ArgRef_VARARGS(__pyx_args, 5);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  5:
        values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  4:
        values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  3:
        values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 546, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }

      /* "cupy/fft/_callback.pyx":547
 * 
 *     def __init__(self,
 *                  cb_load=None,             # <<<<<<<<<<<<<<
 *                  cb_store=None,
 *                  str cb_load_name=None,
*/
      if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":548
 *     def __init__(self,
 *                  cb_load=None,
 *                  cb_store=None,             # <<<<<<<<<<<<<<
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
*/
      if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":549
 *                  cb_load=None,
 *                  cb_store=None,
 *                  str cb_load_name=None,             # <<<<<<<<<<<<<<
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
*/
      if (!values[2]) values[2] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":550
 *                  cb_store=None,
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,
*/
      if (!values[3]) values[3] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":551
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_store_data=None,
 *                  tuple nvrtc_options=(),
*/
      if (!values[4]) values[4] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":552
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,             # <<<<<<<<<<<<<<
 *                  tuple nvrtc_options=(),
 *                  ):
*/
      if (!values[5]) values[5] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":553
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,
 *                  tuple nvrtc_options=(),             # <<<<<<<<<<<<<<
 *                  ):
 *         self._sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
*/
      if (!values[6]) values[6] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_empty_tuple));
    }
    __pyx_v_cb_load = values[0];
    __pyx_v_cb_store = values[1];
    __pyx_v_cb_load_name = ((PyObject*)values[2]);
    __pyx_v_cb_store_name = ((PyObject*)values[3]);
    __pyx_v_cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)values[4]);
    __pyx_v_cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)values[5]);
    __pyx_v_nvrtc_options = ((PyObject*)values[6]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__init__", 0, 0, 7, __pyx_nargs); __PYX_ERR(0, 546, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return -1;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load_name), (&PyUnicode_Type), 1, "cb_load_name", 1))) __PYX_ERR(0, 549, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store_name), (&PyUnicode_Type), 1, "cb_store_name", 1))) __PYX_ERR(0, 550, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load_data), __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer, 1, "cb_load_data", 0))) __PYX_ERR(0, 551, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store_data), __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer, 1, "cb_store_data", 0))) __PYX_ERR(0, 552, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_nvrtc_options), (&PyTuple_Type), 1, "nvrtc_options", 1))) __PYX_ERR(0, 553, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager___init__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self), __pyx_v_cb_load, __pyx_v_cb_store, __pyx_v_cb_load_name, __pyx_v_cb_store_name, __pyx_v_cb_load_data, __pyx_v_cb_store_data, __pyx_v_nvrtc_options);

  /* "cupy/fft/_callback.pyx":546
 *         readonly bytes cb_store_lto
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  cb_load=None,
 *                  cb_store=None,
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = -1;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static int __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager___init__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, PyObject *__pyx_v_cb_load_name, PyObject *__pyx_v_cb_store_name, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data, PyObject *__pyx_v_nvrtc_options) {
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  PyObject *__pyx_t_3 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__init__", 0);

  /* "cupy/fft/_callback.pyx":555
 *                  tuple nvrtc_options=(),
 *                  ):
 *         self._sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)             # <<<<<<<<<<<<<<
 * 
 *         self.identity = "jit"
*/
  __pyx_t_1 = ((struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self->__pyx_base.__pyx_vtab)->_sanity_checks(__pyx_v_self, __pyx_v_cb_load, __pyx_v_cb_store, ((PyObject *)__pyx_v_cb_load_data), ((PyObject *)__pyx_v_cb_store_data)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 555, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":557
 *         self._sanity_checks(cb_load, cb_store, cb_load_data, cb_store_data)
 * 
 *         self.identity = "jit"             # <<<<<<<<<<<<<<
 *         self.cb_load = cb_load
 *         self.cb_store = cb_store
*/
  __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_jit);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_jit);
  __Pyx_GOTREF(__pyx_v_self->__pyx_base.identity);
  __Pyx_DECREF(__pyx_v_self->__pyx_base.identity);
  __pyx_v_self->__pyx_base.identity = __pyx_mstate_global->__pyx_n_u_jit;

  /* "cupy/fft/_callback.pyx":558
 * 
 *         self.identity = "jit"
 *         self.cb_load = cb_load             # <<<<<<<<<<<<<<
 *         self.cb_store = cb_store
 *         self.cb_load_name = cb_load_name
*/
  __Pyx_INCREF(__pyx_v_cb_load);
  __Pyx_GIVEREF(__pyx_v_cb_load);
  __Pyx_GOTREF(__pyx_v_self->cb_load);
  __Pyx_DECREF(__pyx_v_self->cb_load);
  __pyx_v_self->cb_load = __pyx_v_cb_load;

  /* "cupy/fft/_callback.pyx":559
 *         self.identity = "jit"
 *         self.cb_load = cb_load
 *         self.cb_store = cb_store             # <<<<<<<<<<<<<<
 *         self.cb_load_name = cb_load_name
 *         self.cb_store_name = cb_store_name
*/
  __Pyx_INCREF(__pyx_v_cb_store);
  __Pyx_GIVEREF(__pyx_v_cb_store);
  __Pyx_GOTREF(__pyx_v_self->cb_store);
  __Pyx_DECREF(__pyx_v_self->cb_store);
  __pyx_v_self->cb_store = __pyx_v_cb_store;

  /* "cupy/fft/_callback.pyx":560
 *         self.cb_load = cb_load
 *         self.cb_store = cb_store
 *         self.cb_load_name = cb_load_name             # <<<<<<<<<<<<<<
 *         self.cb_store_name = cb_store_name
 *         self.cb_load_data = cb_load_data
*/
  __Pyx_INCREF(__pyx_v_cb_load_name);
  __Pyx_GIVEREF(__pyx_v_cb_load_name);
  __Pyx_GOTREF(__pyx_v_self->cb_load_name);
  __Pyx_DECREF(__pyx_v_self->cb_load_name);
  __pyx_v_self->cb_load_name = __pyx_v_cb_load_name;

  /* "cupy/fft/_callback.pyx":561
 *         self.cb_store = cb_store
 *         self.cb_load_name = cb_load_name
 *         self.cb_store_name = cb_store_name             # <<<<<<<<<<<<<<
 *         self.cb_load_data = cb_load_data
 *         self.cb_store_data = cb_store_data
*/
  __Pyx_INCREF(__pyx_v_cb_store_name);
  __Pyx_GIVEREF(__pyx_v_cb_store_name);
  __Pyx_GOTREF(__pyx_v_self->cb_store_name);
  __Pyx_DECREF(__pyx_v_self->cb_store_name);
  __pyx_v_self->cb_store_name = __pyx_v_cb_store_name;

  /* "cupy/fft/_callback.pyx":562
 *         self.cb_load_name = cb_load_name
 *         self.cb_store_name = cb_store_name
 *         self.cb_load_data = cb_load_data             # <<<<<<<<<<<<<<
 *         self.cb_store_data = cb_store_data
 * 
*/
  __Pyx_INCREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __Pyx_DECREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __pyx_v_self->__pyx_base.cb_load_data = __pyx_v_cb_load_data;

  /* "cupy/fft/_callback.pyx":563
 *         self.cb_store_name = cb_store_name
 *         self.cb_load_data = cb_load_data
 *         self.cb_store_data = cb_store_data             # <<<<<<<<<<<<<<
 * 
 *         if cb_load:
*/
  __Pyx_INCREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __Pyx_DECREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __pyx_v_self->__pyx_base.cb_store_data = __pyx_v_cb_store_data;

  /* "cupy/fft/_callback.pyx":565
 *         self.cb_store_data = cb_store_data
 * 
 *         if cb_load:             # <<<<<<<<<<<<<<
 *             if isinstance(cb_load, str):
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
*/
  __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_cb_load); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 565, __pyx_L1_error)
  if (__pyx_t_2) {

    /* "cupy/fft/_callback.pyx":566
 * 
 *         if cb_load:
 *             if isinstance(cb_load, str):             # <<<<<<<<<<<<<<
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
 *             elif isinstance(cb_load, bytes):
*/
    __pyx_t_2 = PyUnicode_Check(__pyx_v_cb_load); 
    if (__pyx_t_2) {

      /* "cupy/fft/_callback.pyx":567
 *         if cb_load:
 *             if isinstance(cb_load, str):
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)             # <<<<<<<<<<<<<<
 *             elif isinstance(cb_load, bytes):
 *                 self.cb_load_lto = cb_load
*/
      __pyx_t_1 = __pyx_v_cb_load;
      __Pyx_INCREF(__pyx_t_1);
      if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 567, __pyx_L1_error)
      __pyx_t_3 = ((struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self->__pyx_base.__pyx_vtab)->compile_lto(__pyx_v_self, ((PyObject*)__pyx_t_1), __pyx_v_nvrtc_options); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 567, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_3);
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      __Pyx_GIVEREF(__pyx_t_3);
      __Pyx_GOTREF(__pyx_v_self->cb_load_lto);
      __Pyx_DECREF(__pyx_v_self->cb_load_lto);
      __pyx_v_self->cb_load_lto = ((PyObject*)__pyx_t_3);
      __pyx_t_3 = 0;

      /* "cupy/fft/_callback.pyx":566
 * 
 *         if cb_load:
 *             if isinstance(cb_load, str):             # <<<<<<<<<<<<<<
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
 *             elif isinstance(cb_load, bytes):
*/
      goto __pyx_L4;
    }

    /* "cupy/fft/_callback.pyx":568
 *             if isinstance(cb_load, str):
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
 *             elif isinstance(cb_load, bytes):             # <<<<<<<<<<<<<<
 *                 self.cb_load_lto = cb_load
 *             # TODO(leofang): support rawkernel
*/
    __pyx_t_2 = PyBytes_Check(__pyx_v_cb_load); 
    if (likely(__pyx_t_2)) {

      /* "cupy/fft/_callback.pyx":569
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
 *             elif isinstance(cb_load, bytes):
 *                 self.cb_load_lto = cb_load             # <<<<<<<<<<<<<<
 *             # TODO(leofang): support rawkernel
 *             else:
*/
      __pyx_t_3 = __pyx_v_cb_load;
      __Pyx_INCREF(__pyx_t_3);
      if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_3))) __PYX_ERR(0, 569, __pyx_L1_error)
      __Pyx_GIVEREF(__pyx_t_3);
      __Pyx_GOTREF(__pyx_v_self->cb_load_lto);
      __Pyx_DECREF(__pyx_v_self->cb_load_lto);
      __pyx_v_self->cb_load_lto = ((PyObject*)__pyx_t_3);
      __pyx_t_3 = 0;

      /* "cupy/fft/_callback.pyx":568
 *             if isinstance(cb_load, str):
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
 *             elif isinstance(cb_load, bytes):             # <<<<<<<<<<<<<<
 *                 self.cb_load_lto = cb_load
 *             # TODO(leofang): support rawkernel
*/
      goto __pyx_L4;
    }

    /* "cupy/fft/_callback.pyx":572
 *             # TODO(leofang): support rawkernel
 *             else:
 *                 raise NotImplementedError             # <<<<<<<<<<<<<<
 *         if cb_store:
 *             if isinstance(cb_store, str):
*/
    /*else*/ {
      __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_NotImplementedError))), 0, 0, 0);
      __PYX_ERR(0, 572, __pyx_L1_error)
    }
    __pyx_L4:;

    /* "cupy/fft/_callback.pyx":565
 *         self.cb_store_data = cb_store_data
 * 
 *         if cb_load:             # <<<<<<<<<<<<<<
 *             if isinstance(cb_load, str):
 *                 self.cb_load_lto = self.compile_lto(cb_load, nvrtc_options)
*/
  }

  /* "cupy/fft/_callback.pyx":573
 *             else:
 *                 raise NotImplementedError
 *         if cb_store:             # <<<<<<<<<<<<<<
 *             if isinstance(cb_store, str):
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
*/
  __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_cb_store); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 573, __pyx_L1_error)
  if (__pyx_t_2) {

    /* "cupy/fft/_callback.pyx":574
 *                 raise NotImplementedError
 *         if cb_store:
 *             if isinstance(cb_store, str):             # <<<<<<<<<<<<<<
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
 *             elif isinstance(cb_load, bytes):
*/
    __pyx_t_2 = PyUnicode_Check(__pyx_v_cb_store); 
    if (__pyx_t_2) {

      /* "cupy/fft/_callback.pyx":575
 *         if cb_store:
 *             if isinstance(cb_store, str):
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)             # <<<<<<<<<<<<<<
 *             elif isinstance(cb_load, bytes):
 *                 self.cb_store_lto = cb_store
*/
      __pyx_t_3 = __pyx_v_cb_store;
      __Pyx_INCREF(__pyx_t_3);
      if (!(likely(PyUnicode_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_3))) __PYX_ERR(0, 575, __pyx_L1_error)
      __pyx_t_1 = ((struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self->__pyx_base.__pyx_vtab)->compile_lto(__pyx_v_self, ((PyObject*)__pyx_t_3), __pyx_v_nvrtc_options); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 575, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_GIVEREF(__pyx_t_1);
      __Pyx_GOTREF(__pyx_v_self->cb_store_lto);
      __Pyx_DECREF(__pyx_v_self->cb_store_lto);
      __pyx_v_self->cb_store_lto = ((PyObject*)__pyx_t_1);
      __pyx_t_1 = 0;

      /* "cupy/fft/_callback.pyx":574
 *                 raise NotImplementedError
 *         if cb_store:
 *             if isinstance(cb_store, str):             # <<<<<<<<<<<<<<
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
 *             elif isinstance(cb_load, bytes):
*/
      goto __pyx_L6;
    }

    /* "cupy/fft/_callback.pyx":576
 *             if isinstance(cb_store, str):
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
 *             elif isinstance(cb_load, bytes):             # <<<<<<<<<<<<<<
 *                 self.cb_store_lto = cb_store
 *             # TODO(leofang): support rawkernel
*/
    __pyx_t_2 = PyBytes_Check(__pyx_v_cb_load); 
    if (likely(__pyx_t_2)) {

      /* "cupy/fft/_callback.pyx":577
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
 *             elif isinstance(cb_load, bytes):
 *                 self.cb_store_lto = cb_store             # <<<<<<<<<<<<<<
 *             # TODO(leofang): support rawkernel
 *             else:
*/
      __pyx_t_1 = __pyx_v_cb_store;
      __Pyx_INCREF(__pyx_t_1);
      if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_1))) __PYX_ERR(0, 577, __pyx_L1_error)
      __Pyx_GIVEREF(__pyx_t_1);
      __Pyx_GOTREF(__pyx_v_self->cb_store_lto);
      __Pyx_DECREF(__pyx_v_self->cb_store_lto);
      __pyx_v_self->cb_store_lto = ((PyObject*)__pyx_t_1);
      __pyx_t_1 = 0;

      /* "cupy/fft/_callback.pyx":576
 *             if isinstance(cb_store, str):
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
 *             elif isinstance(cb_load, bytes):             # <<<<<<<<<<<<<<
 *                 self.cb_store_lto = cb_store
 *             # TODO(leofang): support rawkernel
*/
      goto __pyx_L6;
    }

    /* "cupy/fft/_callback.pyx":580
 *             # TODO(leofang): support rawkernel
 *             else:
 *                 raise NotImplementedError             # <<<<<<<<<<<<<<
 * 
 *     cdef str _get_cuda_include(self):
*/
    /*else*/ {
      __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_NotImplementedError))), 0, 0, 0);
      __PYX_ERR(0, 580, __pyx_L1_error)
    }
    __pyx_L6:;

    /* "cupy/fft/_callback.pyx":573
 *             else:
 *                 raise NotImplementedError
 *         if cb_store:             # <<<<<<<<<<<<<<
 *             if isinstance(cb_store, str):
 *                 self.cb_store_lto = self.compile_lto(cb_store, nvrtc_options)
*/
  }

  /* "cupy/fft/_callback.pyx":546
 *         readonly bytes cb_store_lto
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  cb_load=None,
 *                  cb_store=None,
*/

  /* function exit code */
  __pyx_r = 0;
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = -1;
  __pyx_L0:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":582
 *                 raise NotImplementedError
 * 
 *     cdef str _get_cuda_include(self):             # <<<<<<<<<<<<<<
 *         global _cuda_include
 *         if _cuda_include is None:
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager__get_cuda_include(CYTHON_UNUSED struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_get_cuda_include", 0);

  /* "cupy/fft/_callback.pyx":584
 *     cdef str _get_cuda_include(self):
 *         global _cuda_include
 *         if _cuda_include is None:             # <<<<<<<<<<<<<<
 *             _cuda_include = find_nvidia_header_directory('cudart')
 *             if _cuda_include is None:
*/
  __pyx_t_1 = (__pyx_v_4cupy_3fft_9_callback__cuda_include == ((PyObject*)Py_None));
  if (__pyx_t_1) {

    /* "cupy/fft/_callback.pyx":585
 *         global _cuda_include
 *         if _cuda_include is None:
 *             _cuda_include = find_nvidia_header_directory('cudart')             # <<<<<<<<<<<<<<
 *             if _cuda_include is None:
 *                 raise RuntimeError('Failed to find CUDA headers.')
*/
    __pyx_t_3 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_find_nvidia_header_directory); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 585, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_5 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_4))) {
      __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
      assert(__pyx_t_3);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
      __Pyx_INCREF(__pyx_t_3);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
      __pyx_t_5 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_mstate_global->__pyx_n_u_cudart};
      __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 585, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
    }
    if (!(likely(PyUnicode_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_2))) __PYX_ERR(0, 585, __pyx_L1_error)
    __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cuda_include);
    __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cuda_include, ((PyObject*)__pyx_t_2));
    __Pyx_GIVEREF(__pyx_t_2);
    __pyx_t_2 = 0;

    /* "cupy/fft/_callback.pyx":586
 *         if _cuda_include is None:
 *             _cuda_include = find_nvidia_header_directory('cudart')
 *             if _cuda_include is None:             # <<<<<<<<<<<<<<
 *                 raise RuntimeError('Failed to find CUDA headers.')
 *         return _cuda_include
*/
    __pyx_t_1 = (__pyx_v_4cupy_3fft_9_callback__cuda_include == ((PyObject*)Py_None));
    if (unlikely(__pyx_t_1)) {

      /* "cupy/fft/_callback.pyx":587
 *             _cuda_include = find_nvidia_header_directory('cudart')
 *             if _cuda_include is None:
 *                 raise RuntimeError('Failed to find CUDA headers.')             # <<<<<<<<<<<<<<
 *         return _cuda_include
 * 
*/
      __pyx_t_4 = NULL;
      __pyx_t_5 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Failed_to_find_CUDA_headers};
        __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
        if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 587, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_2);
      }
      __Pyx_Raise(__pyx_t_2, 0, 0, 0);
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      __PYX_ERR(0, 587, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":586
 *         if _cuda_include is None:
 *             _cuda_include = find_nvidia_header_directory('cudart')
 *             if _cuda_include is None:             # <<<<<<<<<<<<<<
 *                 raise RuntimeError('Failed to find CUDA headers.')
 *         return _cuda_include
*/
    }

    /* "cupy/fft/_callback.pyx":584
 *     cdef str _get_cuda_include(self):
 *         global _cuda_include
 *         if _cuda_include is None:             # <<<<<<<<<<<<<<
 *             _cuda_include = find_nvidia_header_directory('cudart')
 *             if _cuda_include is None:
*/
  }

  /* "cupy/fft/_callback.pyx":588
 *             if _cuda_include is None:
 *                 raise RuntimeError('Failed to find CUDA headers.')
 *         return _cuda_include             # <<<<<<<<<<<<<<
 * 
 *     cdef _sanity_checks(self, cb_load, cb_store, cb_load_data, cb_store_data):
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_4cupy_3fft_9_callback__cuda_include);
  __pyx_r = __pyx_v_4cupy_3fft_9_callback__cuda_include;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":582
 *                 raise NotImplementedError
 * 
 *     cdef str _get_cuda_include(self):             # <<<<<<<<<<<<<<
 *         global _cuda_include
 *         if _cuda_include is None:
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager._get_cuda_include", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":590
 *         return _cuda_include
 * 
 *     cdef _sanity_checks(self, cb_load, cb_store, cb_load_data, cb_store_data):             # <<<<<<<<<<<<<<
 *         from cupy.cuda.cufft import getVersion as get_cufft_version
 *         if runtime._is_hip_environment:
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager__sanity_checks(CYTHON_UNUSED struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, PyObject *__pyx_v_cb_load_data, PyObject *__pyx_v_cb_store_data) {
  PyObject *__pyx_v_get_cufft_version = NULL;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  Py_ssize_t __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  PyObject *__pyx_t_6 = NULL;
  int __pyx_t_7;
  int __pyx_t_8;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("_sanity_checks", 0);

  /* "cupy/fft/_callback.pyx":591
 * 
 *     cdef _sanity_checks(self, cb_load, cb_store, cb_load_data, cb_store_data):
 *         from cupy.cuda.cufft import getVersion as get_cufft_version             # <<<<<<<<<<<<<<
 *         if runtime._is_hip_environment:
 *             raise RuntimeError('hipFFT does not support callbacks')
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_getVersion};
    __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda_cufft, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 591, __pyx_L1_error)
  }
  __pyx_t_1 = __pyx_t_2;
  __Pyx_GOTREF(__pyx_t_1);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_getVersion};
    __pyx_t_3 = 0; {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_1, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 591, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      switch (__pyx_t_3) {
        case 0:
        __Pyx_INCREF(__pyx_t_4);
        __pyx_v_get_cufft_version = __pyx_t_4;
        break;
        default:;
      }
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":592
 *     cdef _sanity_checks(self, cb_load, cb_store, cb_load_data, cb_store_data):
 *         from cupy.cuda.cufft import getVersion as get_cufft_version
 *         if runtime._is_hip_environment:             # <<<<<<<<<<<<<<
 *             raise RuntimeError('hipFFT does not support callbacks')
 *         if get_cufft_version() < 11303:
*/
  if (unlikely(__pyx_v_13cupy_backends_4cuda_3api_7runtime__is_hip_environment)) {

    /* "cupy/fft/_callback.pyx":593
 *         from cupy.cuda.cufft import getVersion as get_cufft_version
 *         if runtime._is_hip_environment:
 *             raise RuntimeError('hipFFT does not support callbacks')             # <<<<<<<<<<<<<<
 *         if get_cufft_version() < 11303:
 *             raise RuntimeError(
*/
    __pyx_t_4 = NULL;
    __pyx_t_5 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_hipFFT_does_not_support_callback};
      __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 593, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_Raise(__pyx_t_1, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __PYX_ERR(0, 593, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":592
 *     cdef _sanity_checks(self, cb_load, cb_store, cb_load_data, cb_store_data):
 *         from cupy.cuda.cufft import getVersion as get_cufft_version
 *         if runtime._is_hip_environment:             # <<<<<<<<<<<<<<
 *             raise RuntimeError('hipFFT does not support callbacks')
 *         if get_cufft_version() < 11303:
*/
  }

  /* "cupy/fft/_callback.pyx":594
 *         if runtime._is_hip_environment:
 *             raise RuntimeError('hipFFT does not support callbacks')
 *         if get_cufft_version() < 11303:             # <<<<<<<<<<<<<<
 *             raise RuntimeError(
 *                 'JIT callback support requires cuFFT from CUDA 12.8+')
*/
  __pyx_t_4 = NULL;
  __Pyx_INCREF(__pyx_v_get_cufft_version);
  __pyx_t_6 = __pyx_v_get_cufft_version; 
  __pyx_t_5 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_6))) {
    __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6);
    assert(__pyx_t_4);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_6, __pyx__function);
    __pyx_t_5 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL};
    __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 594, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __pyx_t_6 = PyObject_RichCompare(__pyx_t_1, __pyx_mstate_global->__pyx_int_11303, Py_LT); __Pyx_XGOTREF(__pyx_t_6); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 594, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_7 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely((__pyx_t_7 < 0))) __PYX_ERR(0, 594, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  if (unlikely(__pyx_t_7)) {

    /* "cupy/fft/_callback.pyx":595
 *             raise RuntimeError('hipFFT does not support callbacks')
 *         if get_cufft_version() < 11303:
 *             raise RuntimeError(             # <<<<<<<<<<<<<<
 *                 'JIT callback support requires cuFFT from CUDA 12.8+')
 *         if cb_load is None and cb_store is None:
*/
    __pyx_t_1 = NULL;
    __pyx_t_5 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_JIT_callback_support_requires_cu};
      __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_RuntimeError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
      if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 595, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_6);
    }
    __Pyx_Raise(__pyx_t_6, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __PYX_ERR(0, 595, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":594
 *         if runtime._is_hip_environment:
 *             raise RuntimeError('hipFFT does not support callbacks')
 *         if get_cufft_version() < 11303:             # <<<<<<<<<<<<<<
 *             raise RuntimeError(
 *                 'JIT callback support requires cuFFT from CUDA 12.8+')
*/
  }

  /* "cupy/fft/_callback.pyx":597
 *             raise RuntimeError(
 *                 'JIT callback support requires cuFFT from CUDA 12.8+')
 *         if cb_load is None and cb_store is None:             # <<<<<<<<<<<<<<
 *             raise ValueError('need to specify either cb_load or cb_store, '
 *                              'or both')
*/
  __pyx_t_8 = (__pyx_v_cb_load == Py_None);
  if (__pyx_t_8) {
  } else {
    __pyx_t_7 = __pyx_t_8;
    goto __pyx_L6_bool_binop_done;
  }
  __pyx_t_8 = (__pyx_v_cb_store == Py_None);
  __pyx_t_7 = __pyx_t_8;
  __pyx_L6_bool_binop_done:;
  if (unlikely(__pyx_t_7)) {

    /* "cupy/fft/_callback.pyx":598
 *                 'JIT callback support requires cuFFT from CUDA 12.8+')
 *         if cb_load is None and cb_store is None:
 *             raise ValueError('need to specify either cb_load or cb_store, '             # <<<<<<<<<<<<<<
 *                              'or both')
 *         if cb_load is not None and cb_store is not None:
*/
    __pyx_t_1 = NULL;
    __pyx_t_5 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_need_to_specify_either_cb_load_o};
      __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
      if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 598, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_6);
    }
    __Pyx_Raise(__pyx_t_6, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __PYX_ERR(0, 598, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":597
 *             raise RuntimeError(
 *                 'JIT callback support requires cuFFT from CUDA 12.8+')
 *         if cb_load is None and cb_store is None:             # <<<<<<<<<<<<<<
 *             raise ValueError('need to specify either cb_load or cb_store, '
 *                              'or both')
*/
  }

  /* "cupy/fft/_callback.pyx":600
 *             raise ValueError('need to specify either cb_load or cb_store, '
 *                              'or both')
 *         if cb_load is not None and cb_store is not None:             # <<<<<<<<<<<<<<
 *             if type(cb_load) is not type(cb_store):
 *                 raise TypeError("when both cb_load and cb_store are given, "
*/
  __pyx_t_8 = (__pyx_v_cb_load != Py_None);
  if (__pyx_t_8) {
  } else {
    __pyx_t_7 = __pyx_t_8;
    goto __pyx_L9_bool_binop_done;
  }
  __pyx_t_8 = (__pyx_v_cb_store != Py_None);
  __pyx_t_7 = __pyx_t_8;
  __pyx_L9_bool_binop_done:;
  if (__pyx_t_7) {

    /* "cupy/fft/_callback.pyx":601
 *                              'or both')
 *         if cb_load is not None and cb_store is not None:
 *             if type(cb_load) is not type(cb_store):             # <<<<<<<<<<<<<<
 *                 raise TypeError("when both cb_load and cb_store are given, "
 *                                 "they must be of the same type")
*/
    __pyx_t_7 = (((PyObject *)Py_TYPE(__pyx_v_cb_load)) != ((PyObject *)Py_TYPE(__pyx_v_cb_store)));
    if (unlikely(__pyx_t_7)) {

      /* "cupy/fft/_callback.pyx":602
 *         if cb_load is not None and cb_store is not None:
 *             if type(cb_load) is not type(cb_store):
 *                 raise TypeError("when both cb_load and cb_store are given, "             # <<<<<<<<<<<<<<
 *                                 "they must be of the same type")
 *         if cb_load_data is not None:
*/
      __pyx_t_1 = NULL;
      __pyx_t_5 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_when_both_cb_load_and_cb_store_a};
        __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
        if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 602, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_6);
      }
      __Pyx_Raise(__pyx_t_6, 0, 0, 0);
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      __PYX_ERR(0, 602, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":601
 *                              'or both')
 *         if cb_load is not None and cb_store is not None:
 *             if type(cb_load) is not type(cb_store):             # <<<<<<<<<<<<<<
 *                 raise TypeError("when both cb_load and cb_store are given, "
 *                                 "they must be of the same type")
*/
    }

    /* "cupy/fft/_callback.pyx":600
 *             raise ValueError('need to specify either cb_load or cb_store, '
 *                              'or both')
 *         if cb_load is not None and cb_store is not None:             # <<<<<<<<<<<<<<
 *             if type(cb_load) is not type(cb_store):
 *                 raise TypeError("when both cb_load and cb_store are given, "
*/
  }

  /* "cupy/fft/_callback.pyx":604
 *                 raise TypeError("when both cb_load and cb_store are given, "
 *                                 "they must be of the same type")
 *         if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *             if cb_load is None:
 *                 raise ValueError('load callback is not given')
*/
  __pyx_t_7 = (__pyx_v_cb_load_data != Py_None);
  if (__pyx_t_7) {

    /* "cupy/fft/_callback.pyx":605
 *                                 "they must be of the same type")
 *         if cb_load_data is not None:
 *             if cb_load is None:             # <<<<<<<<<<<<<<
 *                 raise ValueError('load callback is not given')
 *         if cb_store_data is not None:
*/
    __pyx_t_7 = (__pyx_v_cb_load == Py_None);
    if (unlikely(__pyx_t_7)) {

      /* "cupy/fft/_callback.pyx":606
 *         if cb_load_data is not None:
 *             if cb_load is None:
 *                 raise ValueError('load callback is not given')             # <<<<<<<<<<<<<<
 *         if cb_store_data is not None:
 *             if cb_store is None:
*/
      __pyx_t_1 = NULL;
      __pyx_t_5 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_load_callback_is_not_given};
        __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
        if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 606, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_6);
      }
      __Pyx_Raise(__pyx_t_6, 0, 0, 0);
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      __PYX_ERR(0, 606, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":605
 *                                 "they must be of the same type")
 *         if cb_load_data is not None:
 *             if cb_load is None:             # <<<<<<<<<<<<<<
 *                 raise ValueError('load callback is not given')
 *         if cb_store_data is not None:
*/
    }

    /* "cupy/fft/_callback.pyx":604
 *                 raise TypeError("when both cb_load and cb_store are given, "
 *                                 "they must be of the same type")
 *         if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *             if cb_load is None:
 *                 raise ValueError('load callback is not given')
*/
  }

  /* "cupy/fft/_callback.pyx":607
 *             if cb_load is None:
 *                 raise ValueError('load callback is not given')
 *         if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *             if cb_store is None:
 *                 raise ValueError('store callback is not given')
*/
  __pyx_t_7 = (__pyx_v_cb_store_data != Py_None);
  if (__pyx_t_7) {

    /* "cupy/fft/_callback.pyx":608
 *                 raise ValueError('load callback is not given')
 *         if cb_store_data is not None:
 *             if cb_store is None:             # <<<<<<<<<<<<<<
 *                 raise ValueError('store callback is not given')
 * 
*/
    __pyx_t_7 = (__pyx_v_cb_store == Py_None);
    if (unlikely(__pyx_t_7)) {

      /* "cupy/fft/_callback.pyx":609
 *         if cb_store_data is not None:
 *             if cb_store is None:
 *                 raise ValueError('store callback is not given')             # <<<<<<<<<<<<<<
 * 
 *     cdef bytes compile_lto(self, str source, tuple options):
*/
      __pyx_t_1 = NULL;
      __pyx_t_5 = 1;
      {
        PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_kp_u_store_callback_is_not_given};
        __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
        if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 609, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_6);
      }
      __Pyx_Raise(__pyx_t_6, 0, 0, 0);
      __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
      __PYX_ERR(0, 609, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":608
 *                 raise ValueError('load callback is not given')
 *         if cb_store_data is not None:
 *             if cb_store is None:             # <<<<<<<<<<<<<<
 *                 raise ValueError('store callback is not given')
 * 
*/
    }

    /* "cupy/fft/_callback.pyx":607
 *             if cb_load is None:
 *                 raise ValueError('load callback is not given')
 *         if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *             if cb_store is None:
 *                 raise ValueError('store callback is not given')
*/
  }

  /* "cupy/fft/_callback.pyx":590
 *         return _cuda_include
 * 
 *     cdef _sanity_checks(self, cb_load, cb_store, cb_load_data, cb_store_data):             # <<<<<<<<<<<<<<
 *         from cupy.cuda.cufft import getVersion as get_cufft_version
 *         if runtime._is_hip_environment:
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager._sanity_checks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_get_cufft_version);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":611
 *                 raise ValueError('store callback is not given')
 * 
 *     cdef bytes compile_lto(self, str source, tuple options):             # <<<<<<<<<<<<<<
 *         options += ('--std=c++17', '-I' + self._get_cuda_include())
 *         # TODO(leofang): support log_stream & jitify
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_compile_lto(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_source, PyObject *__pyx_v_options) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  size_t __pyx_t_7;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("compile_lto", 0);
  __Pyx_INCREF(__pyx_v_options);

  /* "cupy/fft/_callback.pyx":612
 * 
 *     cdef bytes compile_lto(self, str source, tuple options):
 *         options += ('--std=c++17', '-I' + self._get_cuda_include())             # <<<<<<<<<<<<<<
 *         # TODO(leofang): support log_stream & jitify
 *         return _compile_with_cache_cuda(
*/
  __pyx_t_1 = ((struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self->__pyx_base.__pyx_vtab)->_get_cuda_include(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 612, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_2 = __Pyx_PyUnicode_ConcatSafe(__pyx_mstate_global->__pyx_kp_u_I, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 612, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 612, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_std_c_17_2);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_kp_u_std_c_17_2);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_mstate_global->__pyx_kp_u_std_c_17_2) != (0)) __PYX_ERR(0, 612, __pyx_L1_error);
  __Pyx_GIVEREF(__pyx_t_2);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 612, __pyx_L1_error);
  __pyx_t_2 = 0;
  __pyx_t_2 = PyNumber_InPlaceAdd(__pyx_v_options, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 612, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_DECREF_SET(__pyx_v_options, ((PyObject*)__pyx_t_2));
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":614
 *         options += ('--std=c++17', '-I' + self._get_cuda_include())
 *         # TODO(leofang): support log_stream & jitify
 *         return _compile_with_cache_cuda(             # <<<<<<<<<<<<<<
 *             source, options, None, get_cache_dir(), to_ltoir=True)
 * 
*/
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_compile_with_cache_cuda); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 614, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);

  /* "cupy/fft/_callback.pyx":615
 *         # TODO(leofang): support log_stream & jitify
 *         return _compile_with_cache_cuda(
 *             source, options, None, get_cache_dir(), to_ltoir=True)             # <<<<<<<<<<<<<<
 * 
 *     cpdef create_plan(self, intptr_t prealloc_plan, tuple plan_info):
*/
  __pyx_t_5 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_get_cache_dir); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 615, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_6))) {
    __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_6);
    assert(__pyx_t_5);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6);
    __Pyx_INCREF(__pyx_t_5);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_6, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_5, NULL};
    __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_7, (1-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 615, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
  }
  __pyx_t_7 = 1;
  #if CYTHON_UNPACK_METHODS
  if (unlikely(PyMethod_Check(__pyx_t_3))) {
    __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_3);
    assert(__pyx_t_1);
    PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3);
    __Pyx_INCREF(__pyx_t_1);
    __Pyx_INCREF(__pyx__function);
    __Pyx_DECREF_SET(__pyx_t_3, __pyx__function);
    __pyx_t_7 = 0;
  }
  #endif
  {
    PyObject *__pyx_callargs[5 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_1, __pyx_v_source, __pyx_v_options, Py_None, __pyx_t_4};
    __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 614, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_6);
    if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_to_ltoir, Py_True, __pyx_t_6, __pyx_callargs+5, 0) < (0)) __PYX_ERR(0, 614, __pyx_L1_error)
    __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_7, (5-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6);
    __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 614, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }

  /* "cupy/fft/_callback.pyx":614
 *         options += ('--std=c++17', '-I' + self._get_cuda_include())
 *         # TODO(leofang): support log_stream & jitify
 *         return _compile_with_cache_cuda(             # <<<<<<<<<<<<<<
 *             source, options, None, get_cache_dir(), to_ltoir=True)
 * 
*/
  if (!(likely(PyBytes_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_2))) __PYX_ERR(0, 614, __pyx_L1_error)
  __pyx_r = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":611
 *                 raise ValueError('store callback is not given')
 * 
 *     cdef bytes compile_lto(self, str source, tuple options):             # <<<<<<<<<<<<<<
 *         options += ('--std=c++17', '-I' + self._get_cuda_include())
 *         # TODO(leofang): support log_stream & jitify
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.compile_lto", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_options);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":617
 *             source, options, None, get_cache_dir(), to_ltoir=True)
 * 
 *     cpdef create_plan(self, intptr_t prealloc_plan, tuple plan_info):             # <<<<<<<<<<<<<<
 *         cdef str plan_type
 *         cdef tuple plan_args
*/

static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_3create_plan(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
static PyObject *__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_create_plan(CYTHON_UNUSED struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, intptr_t __pyx_v_prealloc_plan, PyObject *__pyx_v_plan_info, int __pyx_skip_dispatch) {
  PyObject *__pyx_v_plan_type = 0;
  PyObject *__pyx_v_plan_args = 0;
  PyObject *__pyx_v_cufft = NULL;
  PyObject *__pyx_v_plan = NULL;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  size_t __pyx_t_6;
  PyObject *__pyx_t_7 = NULL;
  Py_ssize_t __pyx_t_8;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("create_plan", 0);
  /* Check if called by wrapper */
  if (unlikely(__pyx_skip_dispatch)) ;
  /* Check if overridden in Python */
  else if (
  #if !CYTHON_USE_TYPE_SLOTS
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self)) != __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager &&
  __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), Py_TPFLAGS_HAVE_GC))
  #else
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0 || __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))
  #endif
  ) {
    #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
    if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) {
      PY_UINT64_T __pyx_typedict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      #endif
      __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_create_plan); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 617, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
      if (!__Pyx_IsSameCFunction(__pyx_t_1, (void(*)(void)) __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_3create_plan)) {
        __Pyx_XDECREF(__pyx_r);
        __pyx_t_3 = NULL;
        __Pyx_INCREF(__pyx_t_1);
        __pyx_t_4 = __pyx_t_1; 
        __pyx_t_5 = PyLong_FromSsize_t(__pyx_v_prealloc_plan); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 617, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_5);
        __pyx_t_6 = 1;
        #if CYTHON_UNPACK_METHODS
        if (unlikely(PyMethod_Check(__pyx_t_4))) {
          __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
          assert(__pyx_t_3);
          PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
          __Pyx_INCREF(__pyx_t_3);
          __Pyx_INCREF(__pyx__function);
          __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
          __pyx_t_6 = 0;
        }
        #endif
        {
          PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_5, __pyx_v_plan_info};
          __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
          if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 617, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_2);
        }
        __pyx_r = __pyx_t_2;
        __pyx_t_2 = 0;
        __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
        goto __pyx_L0;
      }
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
      __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self));
      if (unlikely(__pyx_typedict_guard != __pyx_tp_dict_version)) {
        __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
      }
      #endif
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    }
    #endif
  }

  /* "cupy/fft/_callback.pyx":621
 *         cdef tuple plan_args
 * 
 *         from cupy.cuda import cufft             # <<<<<<<<<<<<<<
 *         plan_type, plan_args = plan_info
 *         plan = getattr(cufft, plan_type)(
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_cufft};
    __pyx_t_7 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 621, __pyx_L1_error)
  }
  __pyx_t_1 = __pyx_t_7;
  __Pyx_GOTREF(__pyx_t_1);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_cufft};
    __pyx_t_8 = 0; {
      __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_imported_names[__pyx_t_8]); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 621, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
      switch (__pyx_t_8) {
        case 0:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_cufft = __pyx_t_2;
        break;
        default:;
      }
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":622
 * 
 *         from cupy.cuda import cufft
 *         plan_type, plan_args = plan_info             # <<<<<<<<<<<<<<
 *         plan = getattr(cufft, plan_type)(
 *             *plan_args, prealloc_plan=prealloc_plan)
*/
  if (likely(__pyx_v_plan_info != Py_None)) {
    PyObject* sequence = __pyx_v_plan_info;
    Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence);
    if (unlikely(size != 2)) {
      if (size > 2) __Pyx_RaiseTooManyValuesError(2);
      else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
      __PYX_ERR(0, 622, __pyx_L1_error)
    }
    #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
    __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0);
    __Pyx_INCREF(__pyx_t_1);
    __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1);
    __Pyx_INCREF(__pyx_t_2);
    #else
    __pyx_t_1 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 622, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 622, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    #endif
  } else {
    __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(0, 622, __pyx_L1_error)
  }
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(0, 622, __pyx_L1_error)
  if (!(likely(PyTuple_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_2))) __PYX_ERR(0, 622, __pyx_L1_error)
  __pyx_v_plan_type = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_v_plan_args = ((PyObject*)__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":623
 *         from cupy.cuda import cufft
 *         plan_type, plan_args = plan_info
 *         plan = getattr(cufft, plan_type)(             # <<<<<<<<<<<<<<
 *             *plan_args, prealloc_plan=prealloc_plan)
 *         return plan
*/
  __pyx_t_2 = __Pyx_GetAttr(__pyx_v_cufft, __pyx_v_plan_type); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 623, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);

  /* "cupy/fft/_callback.pyx":624
 *         plan_type, plan_args = plan_info
 *         plan = getattr(cufft, plan_type)(
 *             *plan_args, prealloc_plan=prealloc_plan)             # <<<<<<<<<<<<<<
 *         return plan
 * 
*/
  if (unlikely(__pyx_v_plan_args == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
    __PYX_ERR(0, 624, __pyx_L1_error)
  }
  __pyx_t_1 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 624, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_t_4 = PyLong_FromSsize_t(__pyx_v_prealloc_plan); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 624, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_prealloc_plan, __pyx_t_4) < (0)) __PYX_ERR(0, 624, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":623
 *         from cupy.cuda import cufft
 *         plan_type, plan_args = plan_info
 *         plan = getattr(cufft, plan_type)(             # <<<<<<<<<<<<<<
 *             *plan_args, prealloc_plan=prealloc_plan)
 *         return plan
*/
  __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_v_plan_args, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 623, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_v_plan = __pyx_t_4;
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":625
 *         plan = getattr(cufft, plan_type)(
 *             *plan_args, prealloc_plan=prealloc_plan)
 *         return plan             # <<<<<<<<<<<<<<
 * 
 *     cpdef intptr_t set_callbacks(self, fft_type) except*:
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_plan);
  __pyx_r = __pyx_v_plan;
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":617
 *             source, options, None, get_cache_dir(), to_ltoir=True)
 * 
 *     cpdef create_plan(self, intptr_t prealloc_plan, tuple plan_info):             # <<<<<<<<<<<<<<
 *         cdef str plan_type
 *         cdef tuple plan_args
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.create_plan", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_plan_type);
  __Pyx_XDECREF(__pyx_v_plan_args);
  __Pyx_XDECREF(__pyx_v_cufft);
  __Pyx_XDECREF(__pyx_v_plan);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_3create_plan(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_2create_plan, "_JITCallbackManager.create_plan(self, intptr_t prealloc_plan, tuple plan_info)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_3create_plan(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  intptr_t __pyx_v_prealloc_plan;
  PyObject *__pyx_v_plan_info = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[2] = {0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("create_plan (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_prealloc_plan,&__pyx_mstate_global->__pyx_n_u_plan_info,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 617, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 617, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 617, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "create_plan", 0) < (0)) __PYX_ERR(0, 617, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 2; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("create_plan", 1, 2, 2, i); __PYX_ERR(0, 617, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 2)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 617, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 617, __pyx_L3_error)
    }
    __pyx_v_prealloc_plan = PyLong_AsSsize_t(values[0]); if (unlikely((__pyx_v_prealloc_plan == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 617, __pyx_L3_error)
    __pyx_v_plan_info = ((PyObject*)values[1]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("create_plan", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 617, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.create_plan", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_plan_info), (&PyTuple_Type), 1, "plan_info", 1))) __PYX_ERR(0, 617, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_2create_plan(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self), __pyx_v_prealloc_plan, __pyx_v_plan_info);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_2create_plan(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, intptr_t __pyx_v_prealloc_plan, PyObject *__pyx_v_plan_info) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("create_plan", 0);
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = __pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_create_plan(__pyx_v_self, __pyx_v_prealloc_plan, __pyx_v_plan_info, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 617, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_r = __pyx_t_1;
  __pyx_t_1 = 0;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.create_plan", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":627
 *         return plan
 * 
 *     cpdef intptr_t set_callbacks(self, fft_type) except*:             # <<<<<<<<<<<<<<
 *         '''Set the load/store callbacks by making calls to
 *         ``cufftXtJITSetCallback``.
*/

static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_5set_callbacks(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
static intptr_t __pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_fft_type, int __pyx_skip_dispatch) {
  PyObject *__pyx_v_CUFFT_C2C = NULL;
  PyObject *__pyx_v_CUFFT_C2R = NULL;
  PyObject *__pyx_v_CUFFT_R2C = NULL;
  PyObject *__pyx_v_CUFFT_Z2Z = NULL;
  PyObject *__pyx_v_CUFFT_Z2D = NULL;
  PyObject *__pyx_v_CUFFT_D2Z = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_COMPLEX = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_REAL = NULL;
  PyObject *__pyx_v_CUFFT_CB_LD_REAL_DOUBLE = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_COMPLEX = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_REAL = NULL;
  PyObject *__pyx_v_CUFFT_CB_ST_REAL_DOUBLE = NULL;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data = 0;
  intptr_t __pyx_v_cb_load_ptr;
  intptr_t __pyx_v_cb_store_ptr;
  PyObject *__pyx_v_cb_load_type = NULL;
  PyObject *__pyx_v_cb_store_type = NULL;
  PyObject *__pyx_v_cufft = NULL;
  intptr_t __pyx_v_plan;
  intptr_t __pyx_r;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  PyObject *__pyx_t_4 = NULL;
  size_t __pyx_t_5;
  intptr_t __pyx_t_6;
  PyObject *__pyx_t_7 = NULL;
  Py_ssize_t __pyx_t_8;
  int __pyx_t_9;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("set_callbacks", 0);
  /* Check if called by wrapper */
  if (unlikely(__pyx_skip_dispatch)) ;
  /* Check if overridden in Python */
  else if (
  #if !CYTHON_USE_TYPE_SLOTS
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self)) != __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager &&
  __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), Py_TPFLAGS_HAVE_GC))
  #else
  unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0 || __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))
  #endif
  ) {
    #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
    if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) {
      PY_UINT64_T __pyx_typedict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      #endif
      __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_set_callbacks); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 627, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
      if (!__Pyx_IsSameCFunction(__pyx_t_1, (void(*)(void)) __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_5set_callbacks)) {
        __pyx_t_3 = NULL;
        __Pyx_INCREF(__pyx_t_1);
        __pyx_t_4 = __pyx_t_1; 
        __pyx_t_5 = 1;
        #if CYTHON_UNPACK_METHODS
        if (unlikely(PyMethod_Check(__pyx_t_4))) {
          __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
          assert(__pyx_t_3);
          PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
          __Pyx_INCREF(__pyx_t_3);
          __Pyx_INCREF(__pyx__function);
          __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
          __pyx_t_5 = 0;
        }
        #endif
        {
          PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_fft_type};
          __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
          __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
          __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
          if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 627, __pyx_L1_error)
          __Pyx_GOTREF(__pyx_t_2);
        }
        __pyx_t_6 = PyLong_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_6 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 627, __pyx_L1_error)
        __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
        __pyx_r = __pyx_t_6;
        __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
        goto __pyx_L0;
      }
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
      __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self));
      __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self));
      if (unlikely(__pyx_typedict_guard != __pyx_tp_dict_version)) {
        __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT;
      }
      #endif
      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
      #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
    }
    #endif
  }

  /* "cupy/fft/_callback.pyx":636
 * 
 *         '''
 *         from cupy.cuda.cufft import (             # <<<<<<<<<<<<<<
 *             CUFFT_C2C, CUFFT_C2R, CUFFT_R2C,
 *             CUFFT_Z2Z, CUFFT_Z2D, CUFFT_D2Z,
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_CUFFT_C2C,__pyx_mstate_global->__pyx_n_u_CUFFT_C2R,__pyx_mstate_global->__pyx_n_u_CUFFT_R2C,__pyx_mstate_global->__pyx_n_u_CUFFT_Z2Z,__pyx_mstate_global->__pyx_n_u_CUFFT_Z2D,__pyx_mstate_global->__pyx_n_u_CUFFT_D2Z,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_COMPLEX,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_COMPLEX_DOUBLE,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_REAL,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_REAL_DOUBLE,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_COMPLEX,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_COMPLEX_DOUBLE,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_REAL,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_REAL_DOUBLE};
    __pyx_t_7 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda_cufft, __pyx_imported_names, 14, NULL, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 636, __pyx_L1_error)
  }
  __pyx_t_1 = __pyx_t_7;
  __Pyx_GOTREF(__pyx_t_1);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_CUFFT_C2C,__pyx_mstate_global->__pyx_n_u_CUFFT_C2R,__pyx_mstate_global->__pyx_n_u_CUFFT_R2C,__pyx_mstate_global->__pyx_n_u_CUFFT_Z2Z,__pyx_mstate_global->__pyx_n_u_CUFFT_Z2D,__pyx_mstate_global->__pyx_n_u_CUFFT_D2Z,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_COMPLEX,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_COMPLEX_DOUBLE,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_REAL,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_LD_REAL_DOUBLE,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_COMPLEX,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_COMPLEX_DOUBLE,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_REAL,__pyx_mstate_global->__pyx_n_u_CUFFT_CB_ST_REAL_DOUBLE};
    for (__pyx_t_8=0; __pyx_t_8 < 14; __pyx_t_8++) {
      __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_imported_names[__pyx_t_8]); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 636, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
      switch (__pyx_t_8) {
        case 0:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_C2C = __pyx_t_2;
        break;
        case 1:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_C2R = __pyx_t_2;
        break;
        case 2:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_R2C = __pyx_t_2;
        break;
        case 3:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_Z2Z = __pyx_t_2;
        break;
        case 4:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_Z2D = __pyx_t_2;
        break;
        case 5:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_D2Z = __pyx_t_2;
        break;
        case 6:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_LD_COMPLEX = __pyx_t_2;
        break;
        case 7:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE = __pyx_t_2;
        break;
        case 8:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_LD_REAL = __pyx_t_2;
        break;
        case 9:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_LD_REAL_DOUBLE = __pyx_t_2;
        break;
        case 10:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_ST_COMPLEX = __pyx_t_2;
        break;
        case 11:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE = __pyx_t_2;
        break;
        case 12:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_ST_REAL = __pyx_t_2;
        break;
        case 13:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_CUFFT_CB_ST_REAL_DOUBLE = __pyx_t_2;
        break;
        default:;
      }
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":644
 *             CUFFT_CB_ST_REAL, CUFFT_CB_ST_REAL_DOUBLE,)
 * 
 *         cdef MemoryPointer cb_load_data = self.cb_load_data             # <<<<<<<<<<<<<<
 *         cdef MemoryPointer cb_store_data = self.cb_store_data
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __Pyx_INCREF(__pyx_t_1);
  __pyx_v_cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":645
 * 
 *         cdef MemoryPointer cb_load_data = self.cb_load_data
 *         cdef MemoryPointer cb_store_data = self.cb_store_data             # <<<<<<<<<<<<<<
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
 * 
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __Pyx_INCREF(__pyx_t_1);
  __pyx_v_cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":646
 *         cdef MemoryPointer cb_load_data = self.cb_load_data
 *         cdef MemoryPointer cb_store_data = self.cb_store_data
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0             # <<<<<<<<<<<<<<
 * 
 *         if fft_type == CUFFT_C2C:
*/
  __pyx_v_cb_load_ptr = 0;
  __pyx_v_cb_store_ptr = 0;

  /* "cupy/fft/_callback.pyx":648
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
 * 
 *         if fft_type == CUFFT_C2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_C2C, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 648, __pyx_L1_error)
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 648, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":649
 * 
 *         if fft_type == CUFFT_C2C:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 649, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":650
 *         if fft_type == CUFFT_C2C:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_R2C:
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 650, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":648
 *         cdef intptr_t cb_load_ptr=0, cb_store_ptr=0
 * 
 *         if fft_type == CUFFT_C2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":651
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_R2C, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 651, __pyx_L1_error)
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 651, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":652
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 652, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_REAL);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_REAL;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":653
 *         elif fft_type == CUFFT_R2C:
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_C2R:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 653, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":651
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_R2C:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":654
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_C2R, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 654, __pyx_L1_error)
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 654, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":655
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 655, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":656
 *         elif fft_type == CUFFT_C2R:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_Z2Z:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 656, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_REAL);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_REAL;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":654
 *             cb_load_type = CUFFT_CB_LD_REAL if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX if self.cb_store else -1
 *         elif fft_type == CUFFT_C2R:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":657
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_Z2Z, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 657, __pyx_L1_error)
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 657, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":658
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 658, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":659
 *         elif fft_type == CUFFT_Z2Z:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_D2Z:
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 659, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":657
 *             cb_load_type = CUFFT_CB_LD_COMPLEX if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":660
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_D2Z, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 660, __pyx_L1_error)
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 660, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":661
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 661, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_REAL_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_REAL_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":662
 *         elif fft_type == CUFFT_D2Z:
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         elif fft_type == CUFFT_Z2D:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 662, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":660
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_D2Z:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":663
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
*/
  __pyx_t_1 = PyObject_RichCompare(__pyx_v_fft_type, __pyx_v_CUFFT_Z2D, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 663, __pyx_L1_error)
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 663, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (likely(__pyx_t_9)) {

    /* "cupy/fft/_callback.pyx":664
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1             # <<<<<<<<<<<<<<
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
 *         else:
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 664, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_load_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":665
 *         elif fft_type == CUFFT_Z2D:
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1             # <<<<<<<<<<<<<<
 *         else:
 *             raise ValueError
*/
    __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 665, __pyx_L1_error)
    if (__pyx_t_9) {
      __Pyx_INCREF(__pyx_v_CUFFT_CB_ST_REAL_DOUBLE);
      __pyx_t_1 = __pyx_v_CUFFT_CB_ST_REAL_DOUBLE;
    } else {
      __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1);
      __pyx_t_1 = __pyx_mstate_global->__pyx_int_neg_1;
    }
    __pyx_v_cb_store_type = __pyx_t_1;
    __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":663
 *             cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if self.cb_store else -1
 *         elif fft_type == CUFFT_Z2D:             # <<<<<<<<<<<<<<
 *             cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if self.cb_load else -1
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
*/
    goto __pyx_L3;
  }

  /* "cupy/fft/_callback.pyx":667
 *             cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if self.cb_store else -1
 *         else:
 *             raise ValueError             # <<<<<<<<<<<<<<
 * 
 *         from cupy.cuda import cufft
*/
  /*else*/ {
    __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), 0, 0, 0);
    __PYX_ERR(0, 667, __pyx_L1_error)
  }
  __pyx_L3:;

  /* "cupy/fft/_callback.pyx":669
 *             raise ValueError
 * 
 *         from cupy.cuda import cufft             # <<<<<<<<<<<<<<
 *         cdef intptr_t plan = cufft.create()
 *         cufft.setAutoAllocation(plan, 0)
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_cufft};
    __pyx_t_7 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 669, __pyx_L1_error)
  }
  __pyx_t_1 = __pyx_t_7;
  __Pyx_GOTREF(__pyx_t_1);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_cufft};
    __pyx_t_8 = 0; {
      __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_imported_names[__pyx_t_8]); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 669, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_2);
      switch (__pyx_t_8) {
        case 0:
        __Pyx_INCREF(__pyx_t_2);
        __pyx_v_cufft = __pyx_t_2;
        break;
        default:;
      }
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":670
 * 
 *         from cupy.cuda import cufft
 *         cdef intptr_t plan = cufft.create()             # <<<<<<<<<<<<<<
 *         cufft.setAutoAllocation(plan, 0)
 * 
*/
  __pyx_t_2 = __pyx_v_cufft;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_5 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_create, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 670, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __pyx_t_6 = PyLong_AsSsize_t(__pyx_t_1); if (unlikely((__pyx_t_6 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 670, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_v_plan = __pyx_t_6;

  /* "cupy/fft/_callback.pyx":671
 *         from cupy.cuda import cufft
 *         cdef intptr_t plan = cufft.create()
 *         cufft.setAutoAllocation(plan, 0)             # <<<<<<<<<<<<<<
 * 
 *         if self.cb_load:
*/
  __pyx_t_2 = __pyx_v_cufft;
  __Pyx_INCREF(__pyx_t_2);
  __pyx_t_4 = PyLong_FromSsize_t(__pyx_v_plan); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 671, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_5 = 0;
  {
    PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_t_4, __pyx_mstate_global->__pyx_int_0};
    __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_setAutoAllocation, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 671, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
  }
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":673
 *         cufft.setAutoAllocation(plan, 0)
 * 
 *         if self.cb_load:             # <<<<<<<<<<<<<<
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr
*/
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_load); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 673, __pyx_L1_error)
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":674
 * 
 *         if self.cb_load:
 *             if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setJITCallback(
*/
    __pyx_t_9 = (((PyObject *)__pyx_v_cb_load_data) != Py_None);
    if (__pyx_t_9) {

      /* "cupy/fft/_callback.pyx":675
 *         if self.cb_load:
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr             # <<<<<<<<<<<<<<
 *             cufft.setJITCallback(
 *                 <intptr_t>plan, self.cb_load_name, self.cb_load_lto,
*/
      __pyx_t_6 = __pyx_v_cb_load_data->ptr;
      __pyx_v_cb_load_ptr = __pyx_t_6;

      /* "cupy/fft/_callback.pyx":674
 * 
 *         if self.cb_load:
 *             if cb_load_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setJITCallback(
*/
    }

    /* "cupy/fft/_callback.pyx":676
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setJITCallback(             # <<<<<<<<<<<<<<
 *                 <intptr_t>plan, self.cb_load_name, self.cb_load_lto,
 *                 cb_load_type, cb_load_ptr)
*/
    __pyx_t_4 = __pyx_v_cufft;
    __Pyx_INCREF(__pyx_t_4);

    /* "cupy/fft/_callback.pyx":677
 *                 cb_load_ptr = cb_load_data.ptr
 *             cufft.setJITCallback(
 *                 <intptr_t>plan, self.cb_load_name, self.cb_load_lto,             # <<<<<<<<<<<<<<
 *                 cb_load_type, cb_load_ptr)
 *         if self.cb_store:
*/
    __pyx_t_2 = PyLong_FromSsize_t(((intptr_t)__pyx_v_plan)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 677, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);

    /* "cupy/fft/_callback.pyx":678
 *             cufft.setJITCallback(
 *                 <intptr_t>plan, self.cb_load_name, self.cb_load_lto,
 *                 cb_load_type, cb_load_ptr)             # <<<<<<<<<<<<<<
 *         if self.cb_store:
 *             if cb_store_data is not None:
*/
    __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_cb_load_ptr); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 678, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_3);
    __pyx_t_5 = 0;
    {
      PyObject *__pyx_callargs[6] = {__pyx_t_4, __pyx_t_2, __pyx_v_self->cb_load_name, __pyx_v_self->cb_load_lto, __pyx_v_cb_load_type, __pyx_t_3};
      __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_setJITCallback, __pyx_callargs+__pyx_t_5, (6-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 676, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":673
 *         cufft.setAutoAllocation(plan, 0)
 * 
 *         if self.cb_load:             # <<<<<<<<<<<<<<
 *             if cb_load_data is not None:
 *                 cb_load_ptr = cb_load_data.ptr
*/
  }

  /* "cupy/fft/_callback.pyx":679
 *                 <intptr_t>plan, self.cb_load_name, self.cb_load_lto,
 *                 cb_load_type, cb_load_ptr)
 *         if self.cb_store:             # <<<<<<<<<<<<<<
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr
*/
  __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_v_self->cb_store); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 679, __pyx_L1_error)
  if (__pyx_t_9) {

    /* "cupy/fft/_callback.pyx":680
 *                 cb_load_type, cb_load_ptr)
 *         if self.cb_store:
 *             if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setJITCallback(
*/
    __pyx_t_9 = (((PyObject *)__pyx_v_cb_store_data) != Py_None);
    if (__pyx_t_9) {

      /* "cupy/fft/_callback.pyx":681
 *         if self.cb_store:
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr             # <<<<<<<<<<<<<<
 *             cufft.setJITCallback(
 *                 <intptr_t>plan, self.cb_store_name, self.cb_store_lto,
*/
      __pyx_t_6 = __pyx_v_cb_store_data->ptr;
      __pyx_v_cb_store_ptr = __pyx_t_6;

      /* "cupy/fft/_callback.pyx":680
 *                 cb_load_type, cb_load_ptr)
 *         if self.cb_store:
 *             if cb_store_data is not None:             # <<<<<<<<<<<<<<
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setJITCallback(
*/
    }

    /* "cupy/fft/_callback.pyx":682
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setJITCallback(             # <<<<<<<<<<<<<<
 *                 <intptr_t>plan, self.cb_store_name, self.cb_store_lto,
 *                 cb_store_type, cb_store_ptr)
*/
    __pyx_t_3 = __pyx_v_cufft;
    __Pyx_INCREF(__pyx_t_3);

    /* "cupy/fft/_callback.pyx":683
 *                 cb_store_ptr = cb_store_data.ptr
 *             cufft.setJITCallback(
 *                 <intptr_t>plan, self.cb_store_name, self.cb_store_lto,             # <<<<<<<<<<<<<<
 *                 cb_store_type, cb_store_ptr)
 * 
*/
    __pyx_t_2 = PyLong_FromSsize_t(((intptr_t)__pyx_v_plan)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 683, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);

    /* "cupy/fft/_callback.pyx":684
 *             cufft.setJITCallback(
 *                 <intptr_t>plan, self.cb_store_name, self.cb_store_lto,
 *                 cb_store_type, cb_store_ptr)             # <<<<<<<<<<<<<<
 * 
 *         return <intptr_t>plan
*/
    __pyx_t_4 = PyLong_FromSsize_t(__pyx_v_cb_store_ptr); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 684, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_5 = 0;
    {
      PyObject *__pyx_callargs[6] = {__pyx_t_3, __pyx_t_2, __pyx_v_self->cb_store_name, __pyx_v_self->cb_store_lto, __pyx_v_cb_store_type, __pyx_t_4};
      __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_setJITCallback, __pyx_callargs+__pyx_t_5, (6-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
      __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
      if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 682, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_1);
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;

    /* "cupy/fft/_callback.pyx":679
 *                 <intptr_t>plan, self.cb_load_name, self.cb_load_lto,
 *                 cb_load_type, cb_load_ptr)
 *         if self.cb_store:             # <<<<<<<<<<<<<<
 *             if cb_store_data is not None:
 *                 cb_store_ptr = cb_store_data.ptr
*/
  }

  /* "cupy/fft/_callback.pyx":686
 *                 cb_store_type, cb_store_ptr)
 * 
 *         return <intptr_t>plan             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __pyx_r = ((intptr_t)__pyx_v_plan);
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":627
 *         return plan
 * 
 *     cpdef intptr_t set_callbacks(self, fft_type) except*:             # <<<<<<<<<<<<<<
 *         '''Set the load/store callbacks by making calls to
 *         ``cufftXtJITSetCallback``.
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.set_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_CUFFT_C2C);
  __Pyx_XDECREF(__pyx_v_CUFFT_C2R);
  __Pyx_XDECREF(__pyx_v_CUFFT_R2C);
  __Pyx_XDECREF(__pyx_v_CUFFT_Z2Z);
  __Pyx_XDECREF(__pyx_v_CUFFT_Z2D);
  __Pyx_XDECREF(__pyx_v_CUFFT_D2Z);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_COMPLEX);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_COMPLEX_DOUBLE);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_REAL);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_LD_REAL_DOUBLE);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_COMPLEX);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_COMPLEX_DOUBLE);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_REAL);
  __Pyx_XDECREF(__pyx_v_CUFFT_CB_ST_REAL_DOUBLE);
  __Pyx_XDECREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_XDECREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_XDECREF(__pyx_v_cb_load_type);
  __Pyx_XDECREF(__pyx_v_cb_store_type);
  __Pyx_XDECREF(__pyx_v_cufft);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_5set_callbacks(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_4set_callbacks, "_JITCallbackManager.set_callbacks(self, fft_type) -> intptr_t\n\nSet the load/store callbacks by making calls to\n``cufftXtJITSetCallback``.\n\n.. note::\n    If :meth:`set_caller_infos` is called, a call to this method must\n    follow.");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_5set_callbacks(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v_fft_type = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("set_callbacks (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_fft_type,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 627, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 627, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "set_callbacks", 0) < (0)) __PYX_ERR(0, 627, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("set_callbacks", 1, 1, 1, i); __PYX_ERR(0, 627, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 627, __pyx_L3_error)
    }
    __pyx_v_fft_type = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("set_callbacks", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 627, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.set_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_4set_callbacks(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self), __pyx_v_fft_type);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_4set_callbacks(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v_fft_type) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  intptr_t __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("set_callbacks", 0);
  __Pyx_XDECREF(__pyx_r);
  __pyx_t_1 = __pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_set_callbacks(__pyx_v_self, __pyx_v_fft_type, 1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 627, __pyx_L1_error)
  __pyx_t_2 = PyLong_FromSsize_t(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 627, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __pyx_r = __pyx_t_2;
  __pyx_t_2 = 0;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.set_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":539
 * 
 *     cdef:
 *         readonly object cb_load             # <<<<<<<<<<<<<<
 *         readonly object cb_store
 *         readonly str cb_load_name
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_7cb_load_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_7cb_load_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_7cb_load___get__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_7cb_load___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_load);
  __pyx_r = __pyx_v_self->cb_load;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":540
 *     cdef:
 *         readonly object cb_load
 *         readonly object cb_store             # <<<<<<<<<<<<<<
 *         readonly str cb_load_name
 *         readonly str cb_store_name
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_8cb_store_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_8cb_store_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_8cb_store___get__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_8cb_store___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_store);
  __pyx_r = __pyx_v_self->cb_store;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":541
 *         readonly object cb_load
 *         readonly object cb_store
 *         readonly str cb_load_name             # <<<<<<<<<<<<<<
 *         readonly str cb_store_name
 *         readonly bytes cb_load_lto
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_load_name_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_load_name_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_load_name___get__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_load_name___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_load_name);
  __pyx_r = __pyx_v_self->cb_load_name;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":542
 *         readonly object cb_store
 *         readonly str cb_load_name
 *         readonly str cb_store_name             # <<<<<<<<<<<<<<
 *         readonly bytes cb_load_lto
 *         readonly bytes cb_store_lto
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_13cb_store_name_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_13cb_store_name_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_13cb_store_name___get__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_13cb_store_name___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_store_name);
  __pyx_r = __pyx_v_self->cb_store_name;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":543
 *         readonly str cb_load_name
 *         readonly str cb_store_name
 *         readonly bytes cb_load_lto             # <<<<<<<<<<<<<<
 *         readonly bytes cb_store_lto
 * 
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_11cb_load_lto_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_11cb_load_lto_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_11cb_load_lto___get__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_11cb_load_lto___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_load_lto);
  __pyx_r = __pyx_v_self->cb_load_lto;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":544
 *         readonly str cb_store_name
 *         readonly bytes cb_load_lto
 *         readonly bytes cb_store_lto             # <<<<<<<<<<<<<<
 * 
 *     def __init__(self,
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_store_lto_1__get__(PyObject *__pyx_v_self); /*proto*/
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_store_lto_1__get__(PyObject *__pyx_v_self) {
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_store_lto___get__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_store_lto___get__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__get__", 0);
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v_self->cb_store_lto);
  __pyx_r = __pyx_v_self->cb_store_lto;
  goto __pyx_L0;

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_7__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_6__reduce_cython__, "_JITCallbackManager.__reduce_cython__(self)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_7__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return NULL;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_6__reduce_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_6__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self) {
  PyObject *__pyx_v_state = 0;
  PyObject *__pyx_v__dict = 0;
  int __pyx_v_use_setstate;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__reduce_cython__", 0);

  /* "(tree fragment)":5
 *     cdef object _dict
 *     cdef bint use_setstate
 *     state = (self.cb_load, self.cb_load_data, self.cb_load_lto, self.cb_load_name, self.cb_store, self.cb_store_data, self.cb_store_lto, self.cb_store_name, self.identity)             # <<<<<<<<<<<<<<
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
*/
  __pyx_t_1 = PyTuple_New(9); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_INCREF(__pyx_v_self->cb_load);
  __Pyx_GIVEREF(__pyx_v_self->cb_load);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->cb_load) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->__pyx_base.cb_load_data);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_self->__pyx_base.cb_load_data)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->cb_load_lto);
  __Pyx_GIVEREF(__pyx_v_self->cb_load_lto);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_self->cb_load_lto) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->cb_load_name);
  __Pyx_GIVEREF(__pyx_v_self->cb_load_name);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_v_self->cb_load_name) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->cb_store);
  __Pyx_GIVEREF(__pyx_v_self->cb_store);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 4, __pyx_v_self->cb_store) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->__pyx_base.cb_store_data);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 5, ((PyObject *)__pyx_v_self->__pyx_base.cb_store_data)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->cb_store_lto);
  __Pyx_GIVEREF(__pyx_v_self->cb_store_lto);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 6, __pyx_v_self->cb_store_lto) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->cb_store_name);
  __Pyx_GIVEREF(__pyx_v_self->cb_store_name);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 7, __pyx_v_self->cb_store_name) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_self->__pyx_base.identity);
  __Pyx_GIVEREF(__pyx_v_self->__pyx_base.identity);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 8, __pyx_v_self->__pyx_base.identity) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __pyx_v_state = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":6
 *     cdef bint use_setstate
 *     state = (self.cb_load, self.cb_load_data, self.cb_load_lto, self.cb_load_name, self.cb_store, self.cb_store_data, self.cb_store_lto, self.cb_store_name, self.identity)
 *     _dict = getattr(self, '__dict__', None)             # <<<<<<<<<<<<<<
 *     if _dict is not None and _dict:
 *         state += (_dict,)
*/
  __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 6, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v__dict = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "(tree fragment)":7
 *     state = (self.cb_load, self.cb_load_data, self.cb_load_lto, self.cb_load_name, self.cb_store, self.cb_store_data, self.cb_store_lto, self.cb_store_name, self.identity)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
  __pyx_t_3 = (__pyx_v__dict != Py_None);
  if (__pyx_t_3) {
  } else {
    __pyx_t_2 = __pyx_t_3;
    goto __pyx_L4_bool_binop_done;
  }
  __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(3, 7, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_3;
  __pyx_L4_bool_binop_done:;
  if (__pyx_t_2) {

    /* "(tree fragment)":8
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
 *         state += (_dict,)             # <<<<<<<<<<<<<<
 *         use_setstate = True
 *     else:
*/
    __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(__pyx_v__dict);
    __Pyx_GIVEREF(__pyx_v__dict);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(3, 8, __pyx_L1_error);
    __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "(tree fragment)":9
 *     if _dict is not None and _dict:
 *         state += (_dict,)
 *         use_setstate = True             # <<<<<<<<<<<<<<
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_load_lto is not None or self.cb_load_name is not None or self.cb_store is not None or self.cb_store_data is not None or self.cb_store_lto is not None or self.cb_store_name is not None or self.identity is not None
*/
    __pyx_v_use_setstate = 1;

    /* "(tree fragment)":7
 *     state = (self.cb_load, self.cb_load_data, self.cb_load_lto, self.cb_load_name, self.cb_store, self.cb_store_data, self.cb_store_lto, self.cb_store_name, self.identity)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
    goto __pyx_L3;
  }

  /* "(tree fragment)":11
 *         use_setstate = True
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_load_lto is not None or self.cb_load_name is not None or self.cb_store is not None or self.cb_store_data is not None or self.cb_store_lto is not None or self.cb_store_name is not None or self.identity is not None             # <<<<<<<<<<<<<<
 *     if use_setstate:
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, None), state
*/
  /*else*/ {
    __pyx_t_3 = (__pyx_v_self->cb_load != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (((PyObject *)__pyx_v_self->__pyx_base.cb_load_data) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->cb_load_lto != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->cb_load_name != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->cb_store != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (((PyObject *)__pyx_v_self->__pyx_base.cb_store_data) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->cb_store_lto != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->cb_store_name != ((PyObject*)Py_None));
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (__pyx_v_self->__pyx_base.identity != ((PyObject*)Py_None));
    __pyx_t_2 = __pyx_t_3;
    __pyx_L6_bool_binop_done:;
    __pyx_v_use_setstate = __pyx_t_2;
  }
  __pyx_L3:;

  /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_load_lto is not None or self.cb_load_name is not None or self.cb_store is not None or self.cb_store_data is not None or self.cb_store_lto is not None or self.cb_store_name is not None or self.identity is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, None), state
 *     else:
*/
  if (__pyx_v_use_setstate) {

    /* "(tree fragment)":13
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_load_lto is not None or self.cb_load_name is not None or self.cb_store is not None or self.cb_store_data is not None or self.cb_store_lto is not None or self.cb_store_name is not None or self.identity is not None
 *     if use_setstate:
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, None), state             # <<<<<<<<<<<<<<
 *     else:
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, state)
*/
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__JITCallbackManag); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_238673578);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_238673578);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_238673578) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(Py_None);
    __Pyx_GIVEREF(Py_None);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_GIVEREF(__pyx_t_4);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_4 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_5;
    __pyx_t_5 = 0;
    goto __pyx_L0;

    /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.cb_load is not None or self.cb_load_data is not None or self.cb_load_lto is not None or self.cb_load_name is not None or self.cb_store is not None or self.cb_store_data is not None or self.cb_store_lto is not None or self.cb_store_name is not None or self.identity is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, None), state
 *     else:
*/
  }

  /* "(tree fragment)":15
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, None), state
 *     else:
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, state)             # <<<<<<<<<<<<<<
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__JITCallbackManager__set_state(self, __pyx_state)
*/
  /*else*/ {
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__JITCallbackManag); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_238673578);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_238673578);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_238673578) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_GIVEREF(__pyx_t_5);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_5 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_4;
    __pyx_t_4 = 0;
    goto __pyx_L0;
  }

  /* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_state);
  __Pyx_XDECREF(__pyx_v__dict);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__JITCallbackManager__set_state(self, __pyx_state)
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_9__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_8__setstate_cython__, "_JITCallbackManager.__setstate_cython__(self, __pyx_state)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_9__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 16, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(3, 16, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(3, 16, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
    }
    __pyx_v___pyx_state = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(3, 16, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_8__setstate_cython__(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v_self), __pyx_v___pyx_state);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19_JITCallbackManager_8__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v_self, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__setstate_cython__", 0);

  /* "(tree fragment)":17
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, state)
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle__JITCallbackManager__set_state(self, __pyx_state)             # <<<<<<<<<<<<<<
*/
  __pyx_t_1 = __pyx_v___pyx_state;
  __Pyx_INCREF(__pyx_t_1);
  if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(3, 17, __pyx_L1_error)
  if (unlikely(__pyx_t_1 == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
    __PYX_ERR(3, 17, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__JITCallbackManager__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 17, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle__JITCallbackManager, (type(self), 0xe39deaa, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle__JITCallbackManager__set_state(self, __pyx_state)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_AddTraceback("cupy.fft._callback._JITCallbackManager.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":794
 *         _CallbackManager mgr_prev
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  cb_load=None,
 *                  cb_store=None,
*/

/* Python wrapper */
static int __pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static int __pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
  PyObject *__pyx_v_cb_load = 0;
  PyObject *__pyx_v_cb_store = 0;
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_v_cb_load_aux_arr = 0;
  struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_v_cb_store_aux_arr = 0;
  PyObject *__pyx_v_cb_load_name = 0;
  PyObject *__pyx_v_cb_store_name = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data = 0;
  struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data = 0;
  PyObject *__pyx_v_cb_ver = 0;
  PyObject *__pyx_v_nvrtc_options = 0;
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[10] = {0,0,0,0,0,0,0,0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1;
  #endif
  __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_cb_load,&__pyx_mstate_global->__pyx_n_u_cb_store,&__pyx_mstate_global->__pyx_n_u_cb_load_aux_arr,&__pyx_mstate_global->__pyx_n_u_cb_store_aux_arr,&__pyx_mstate_global->__pyx_n_u_cb_load_name,&__pyx_mstate_global->__pyx_n_u_cb_store_name,&__pyx_mstate_global->__pyx_n_u_cb_load_data,&__pyx_mstate_global->__pyx_n_u_cb_store_data,&__pyx_mstate_global->__pyx_n_u_cb_ver,&__pyx_mstate_global->__pyx_n_u_nvrtc_options,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 794, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  2:
        values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 794, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 794, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(0, 794, __pyx_L3_error)

      /* "cupy/fft/_callback.pyx":795
 * 
 *     def __init__(self,
 *                  cb_load=None,             # <<<<<<<<<<<<<<
 *                  cb_store=None,
 *                  *,
*/
      if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":796
 *     def __init__(self,
 *                  cb_load=None,
 *                  cb_store=None,             # <<<<<<<<<<<<<<
 *                  *,
 *                  _ndarray_base cb_load_aux_arr=None,
*/
      if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":798
 *                  cb_store=None,
 *                  *,
 *                  _ndarray_base cb_load_aux_arr=None,             # <<<<<<<<<<<<<<
 *                  _ndarray_base cb_store_aux_arr=None,
 *                  str cb_load_name=None,
*/
      if (!values[2]) values[2] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)Py_None));

      /* "cupy/fft/_callback.pyx":799
 *                  *,
 *                  _ndarray_base cb_load_aux_arr=None,
 *                  _ndarray_base cb_store_aux_arr=None,             # <<<<<<<<<<<<<<
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
*/
      if (!values[3]) values[3] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)Py_None));

      /* "cupy/fft/_callback.pyx":800
 *                  _ndarray_base cb_load_aux_arr=None,
 *                  _ndarray_base cb_store_aux_arr=None,
 *                  str cb_load_name=None,             # <<<<<<<<<<<<<<
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
*/
      if (!values[4]) values[4] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":801
 *                  _ndarray_base cb_store_aux_arr=None,
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,
*/
      if (!values[5]) values[5] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":802
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_store_data=None,
 *                  str cb_ver='legacy',
*/
      if (!values[6]) values[6] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":803
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,             # <<<<<<<<<<<<<<
 *                  str cb_ver='legacy',
 *                  tuple nvrtc_options=()):
*/
      if (!values[7]) values[7] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));
      if (!values[8]) values[8] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_n_u_legacy));

      /* "cupy/fft/_callback.pyx":805
 *                  MemoryPointer cb_store_data=None,
 *                  str cb_ver='legacy',
 *                  tuple nvrtc_options=()):             # <<<<<<<<<<<<<<
 *         if cb_ver not in ('legacy', 'jit'):
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
*/
      if (!values[9]) values[9] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_empty_tuple));
    } else {
      switch (__pyx_nargs) {
        case  2:
        values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 794, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 794, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }

      /* "cupy/fft/_callback.pyx":795
 * 
 *     def __init__(self,
 *                  cb_load=None,             # <<<<<<<<<<<<<<
 *                  cb_store=None,
 *                  *,
*/
      if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":796
 *     def __init__(self,
 *                  cb_load=None,
 *                  cb_store=None,             # <<<<<<<<<<<<<<
 *                  *,
 *                  _ndarray_base cb_load_aux_arr=None,
*/
      if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)Py_None));

      /* "cupy/fft/_callback.pyx":798
 *                  cb_store=None,
 *                  *,
 *                  _ndarray_base cb_load_aux_arr=None,             # <<<<<<<<<<<<<<
 *                  _ndarray_base cb_store_aux_arr=None,
 *                  str cb_load_name=None,
*/
      if (!values[2]) values[2] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)Py_None));

      /* "cupy/fft/_callback.pyx":799
 *                  *,
 *                  _ndarray_base cb_load_aux_arr=None,
 *                  _ndarray_base cb_store_aux_arr=None,             # <<<<<<<<<<<<<<
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
*/
      if (!values[3]) values[3] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)Py_None));

      /* "cupy/fft/_callback.pyx":800
 *                  _ndarray_base cb_load_aux_arr=None,
 *                  _ndarray_base cb_store_aux_arr=None,
 *                  str cb_load_name=None,             # <<<<<<<<<<<<<<
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
*/
      if (!values[4]) values[4] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":801
 *                  _ndarray_base cb_store_aux_arr=None,
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,
*/
      if (!values[5]) values[5] = __Pyx_NewRef(((PyObject*)Py_None));

      /* "cupy/fft/_callback.pyx":802
 *                  str cb_load_name=None,
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,             # <<<<<<<<<<<<<<
 *                  MemoryPointer cb_store_data=None,
 *                  str cb_ver='legacy',
*/
      if (!values[6]) values[6] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));

      /* "cupy/fft/_callback.pyx":803
 *                  str cb_store_name=None,
 *                  MemoryPointer cb_load_data=None,
 *                  MemoryPointer cb_store_data=None,             # <<<<<<<<<<<<<<
 *                  str cb_ver='legacy',
 *                  tuple nvrtc_options=()):
*/
      if (!values[7]) values[7] = __Pyx_NewRef((PyObject *)((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None));
      if (!values[8]) values[8] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_n_u_legacy));

      /* "cupy/fft/_callback.pyx":805
 *                  MemoryPointer cb_store_data=None,
 *                  str cb_ver='legacy',
 *                  tuple nvrtc_options=()):             # <<<<<<<<<<<<<<
 *         if cb_ver not in ('legacy', 'jit'):
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
*/
      if (!values[9]) values[9] = __Pyx_NewRef(((PyObject*)__pyx_mstate_global->__pyx_empty_tuple));
    }
    __pyx_v_cb_load = values[0];
    __pyx_v_cb_store = values[1];
    __pyx_v_cb_load_aux_arr = ((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)values[2]);
    __pyx_v_cb_store_aux_arr = ((struct __pyx_obj_4cupy_5_core_4core__ndarray_base *)values[3]);
    __pyx_v_cb_load_name = ((PyObject*)values[4]);
    __pyx_v_cb_store_name = ((PyObject*)values[5]);
    __pyx_v_cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)values[6]);
    __pyx_v_cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)values[7]);
    __pyx_v_cb_ver = ((PyObject*)values[8]);
    __pyx_v_nvrtc_options = ((PyObject*)values[9]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__init__", 0, 0, 2, __pyx_nargs); __PYX_ERR(0, 794, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return -1;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load_aux_arr), __pyx_mstate_global->__pyx_ptype_4cupy_5_core_4core__ndarray_base, 1, "cb_load_aux_arr", 0))) __PYX_ERR(0, 798, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store_aux_arr), __pyx_mstate_global->__pyx_ptype_4cupy_5_core_4core__ndarray_base, 1, "cb_store_aux_arr", 0))) __PYX_ERR(0, 799, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load_name), (&PyUnicode_Type), 1, "cb_load_name", 1))) __PYX_ERR(0, 800, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store_name), (&PyUnicode_Type), 1, "cb_store_name", 1))) __PYX_ERR(0, 801, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_load_data), __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer, 1, "cb_load_data", 0))) __PYX_ERR(0, 802, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_store_data), __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer, 1, "cb_store_data", 0))) __PYX_ERR(0, 803, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_cb_ver), (&PyUnicode_Type), 1, "cb_ver", 1))) __PYX_ERR(0, 804, __pyx_L1_error)
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_nvrtc_options), (&PyTuple_Type), 1, "nvrtc_options", 1))) __PYX_ERR(0, 805, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks___init__(((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)__pyx_v_self), __pyx_v_cb_load, __pyx_v_cb_store, __pyx_v_cb_load_aux_arr, __pyx_v_cb_store_aux_arr, __pyx_v_cb_load_name, __pyx_v_cb_store_name, __pyx_v_cb_load_data, __pyx_v_cb_store_data, __pyx_v_cb_ver, __pyx_v_nvrtc_options);

  /* "cupy/fft/_callback.pyx":794
 *         _CallbackManager mgr_prev
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  cb_load=None,
 *                  cb_store=None,
*/

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = -1;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static int __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks___init__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self, PyObject *__pyx_v_cb_load, PyObject *__pyx_v_cb_store, struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_v_cb_load_aux_arr, struct __pyx_obj_4cupy_5_core_4core__ndarray_base *__pyx_v_cb_store_aux_arr, PyObject *__pyx_v_cb_load_name, PyObject *__pyx_v_cb_store_name, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_load_data, struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *__pyx_v_cb_store_data, PyObject *__pyx_v_cb_ver, PyObject *__pyx_v_nvrtc_options) {
  PyObject *__pyx_v_key = 0;
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v_mgr = 0;
  int __pyx_r;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  size_t __pyx_t_6;
  PyObject *__pyx_t_7 = NULL;
  PyObject *__pyx_t_8 = NULL;
  PyObject *__pyx_t_9 = NULL;
  PyObject *__pyx_t_10 = NULL;
  PyObject *__pyx_t_11 = NULL;
  PyObject *__pyx_t_12 = NULL;
  int __pyx_t_13;
  struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos __pyx_t_14;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__init__", 0);
  __Pyx_INCREF(__pyx_v_cb_load_name);
  __Pyx_INCREF(__pyx_v_cb_store_name);
  __Pyx_INCREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_INCREF((PyObject *)__pyx_v_cb_store_data);

  /* "cupy/fft/_callback.pyx":806
 *                  str cb_ver='legacy',
 *                  tuple nvrtc_options=()):
 *         if cb_ver not in ('legacy', 'jit'):             # <<<<<<<<<<<<<<
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
 *         if cb_load_aux_arr is not None or cb_store_aux_arr is not None:
*/
  __Pyx_INCREF(__pyx_v_cb_ver);
  __pyx_t_1 = __pyx_v_cb_ver;
  __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_legacy, Py_NE)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 806, __pyx_L1_error)
  if (__pyx_t_3) {
  } else {
    __pyx_t_2 = __pyx_t_3;
    goto __pyx_L4_bool_binop_done;
  }
  __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_jit, Py_NE)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 806, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_3;
  __pyx_L4_bool_binop_done:;
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_3 = __pyx_t_2;
  if (unlikely(__pyx_t_3)) {

    /* "cupy/fft/_callback.pyx":807
 *                  tuple nvrtc_options=()):
 *         if cb_ver not in ('legacy', 'jit'):
 *             raise ValueError('cb_ver must be "legacy" or "jit"')             # <<<<<<<<<<<<<<
 *         if cb_load_aux_arr is not None or cb_store_aux_arr is not None:
 *             warnings.warn(
*/
    __pyx_t_5 = NULL;
    __pyx_t_6 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_cb_ver_must_be_legacy_or_jit};
      __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 807, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __Pyx_Raise(__pyx_t_4, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __PYX_ERR(0, 807, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":806
 *                  str cb_ver='legacy',
 *                  tuple nvrtc_options=()):
 *         if cb_ver not in ('legacy', 'jit'):             # <<<<<<<<<<<<<<
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
 *         if cb_load_aux_arr is not None or cb_store_aux_arr is not None:
*/
  }

  /* "cupy/fft/_callback.pyx":808
 *         if cb_ver not in ('legacy', 'jit'):
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
 *         if cb_load_aux_arr is not None or cb_store_aux_arr is not None:             # <<<<<<<<<<<<<<
 *             warnings.warn(
 *                 'passing an ndarray to cb_load_aux_arr or cb_store_aux_arr '
*/
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_load_aux_arr) != Py_None);
  if (!__pyx_t_2) {
  } else {
    __pyx_t_3 = __pyx_t_2;
    goto __pyx_L7_bool_binop_done;
  }
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_store_aux_arr) != Py_None);
  __pyx_t_3 = __pyx_t_2;
  __pyx_L7_bool_binop_done:;
  if (__pyx_t_3) {

    /* "cupy/fft/_callback.pyx":809
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
 *         if cb_load_aux_arr is not None or cb_store_aux_arr is not None:
 *             warnings.warn(             # <<<<<<<<<<<<<<
 *                 'passing an ndarray to cb_load_aux_arr or cb_store_aux_arr '
 *                 'is deprecated, please switch to pass arr.data to cb_load_data'
*/
    __pyx_t_5 = NULL;
    __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_warnings); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 809, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_7);
    __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_warn); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 809, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_8);
    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;

    /* "cupy/fft/_callback.pyx":813
 *                 'is deprecated, please switch to pass arr.data to cb_load_data'
 *                 ' or cb_store_data instead, the *_aux_arr arguments will be '
 *                 'removed in a future release', DeprecationWarning)             # <<<<<<<<<<<<<<
 *         if cb_load_aux_arr is not None and cb_load_data is not None:
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")
*/
    __pyx_t_6 = 1;
    #if CYTHON_UNPACK_METHODS
    if (unlikely(PyMethod_Check(__pyx_t_8))) {
      __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_8);
      assert(__pyx_t_5);
      PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8);
      __Pyx_INCREF(__pyx_t_5);
      __Pyx_INCREF(__pyx__function);
      __Pyx_DECREF_SET(__pyx_t_8, __pyx__function);
      __pyx_t_6 = 0;
    }
    #endif
    {
      PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_passing_an_ndarray_to_cb_load_au, ((PyObject *)(((PyTypeObject*)PyExc_DeprecationWarning)))};
      __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
      __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 809, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;

    /* "cupy/fft/_callback.pyx":808
 *         if cb_ver not in ('legacy', 'jit'):
 *             raise ValueError('cb_ver must be "legacy" or "jit"')
 *         if cb_load_aux_arr is not None or cb_store_aux_arr is not None:             # <<<<<<<<<<<<<<
 *             warnings.warn(
 *                 'passing an ndarray to cb_load_aux_arr or cb_store_aux_arr '
*/
  }

  /* "cupy/fft/_callback.pyx":814
 *                 ' or cb_store_data instead, the *_aux_arr arguments will be '
 *                 'removed in a future release', DeprecationWarning)
 *         if cb_load_aux_arr is not None and cb_load_data is not None:             # <<<<<<<<<<<<<<
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")
 *         if cb_store_aux_arr is not None and cb_store_data is not None:
*/
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_load_aux_arr) != Py_None);
  if (__pyx_t_2) {
  } else {
    __pyx_t_3 = __pyx_t_2;
    goto __pyx_L10_bool_binop_done;
  }
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_load_data) != Py_None);
  __pyx_t_3 = __pyx_t_2;
  __pyx_L10_bool_binop_done:;
  if (unlikely(__pyx_t_3)) {

    /* "cupy/fft/_callback.pyx":815
 *                 'removed in a future release', DeprecationWarning)
 *         if cb_load_aux_arr is not None and cb_load_data is not None:
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")             # <<<<<<<<<<<<<<
 *         if cb_store_aux_arr is not None and cb_store_data is not None:
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")
*/
    __pyx_t_8 = NULL;
    __pyx_t_6 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_mstate_global->__pyx_kp_u_only_use_cb_load_data_and_not_cb};
      __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 815, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __Pyx_Raise(__pyx_t_4, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __PYX_ERR(0, 815, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":814
 *                 ' or cb_store_data instead, the *_aux_arr arguments will be '
 *                 'removed in a future release', DeprecationWarning)
 *         if cb_load_aux_arr is not None and cb_load_data is not None:             # <<<<<<<<<<<<<<
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")
 *         if cb_store_aux_arr is not None and cb_store_data is not None:
*/
  }

  /* "cupy/fft/_callback.pyx":816
 *         if cb_load_aux_arr is not None and cb_load_data is not None:
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")
 *         if cb_store_aux_arr is not None and cb_store_data is not None:             # <<<<<<<<<<<<<<
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")
 *         if cb_load_data is None and cb_load_aux_arr is not None:
*/
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_store_aux_arr) != Py_None);
  if (__pyx_t_2) {
  } else {
    __pyx_t_3 = __pyx_t_2;
    goto __pyx_L13_bool_binop_done;
  }
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_store_data) != Py_None);
  __pyx_t_3 = __pyx_t_2;
  __pyx_L13_bool_binop_done:;
  if (unlikely(__pyx_t_3)) {

    /* "cupy/fft/_callback.pyx":817
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")
 *         if cb_store_aux_arr is not None and cb_store_data is not None:
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")             # <<<<<<<<<<<<<<
 *         if cb_load_data is None and cb_load_aux_arr is not None:
 *             cb_load_data = cb_load_aux_arr.data
*/
    __pyx_t_8 = NULL;
    __pyx_t_6 = 1;
    {
      PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_mstate_global->__pyx_kp_u_only_use_cb_store_data_and_not_c};
      __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
      __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
      if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 817, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
    }
    __Pyx_Raise(__pyx_t_4, 0, 0, 0);
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    __PYX_ERR(0, 817, __pyx_L1_error)

    /* "cupy/fft/_callback.pyx":816
 *         if cb_load_aux_arr is not None and cb_load_data is not None:
 *             raise ValueError("only use cb_load_data and not cb_load_aux_arr")
 *         if cb_store_aux_arr is not None and cb_store_data is not None:             # <<<<<<<<<<<<<<
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")
 *         if cb_load_data is None and cb_load_aux_arr is not None:
*/
  }

  /* "cupy/fft/_callback.pyx":818
 *         if cb_store_aux_arr is not None and cb_store_data is not None:
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")
 *         if cb_load_data is None and cb_load_aux_arr is not None:             # <<<<<<<<<<<<<<
 *             cb_load_data = cb_load_aux_arr.data
 *         if cb_store_data is None and cb_store_aux_arr is not None:
*/
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_load_data) == Py_None);
  if (__pyx_t_2) {
  } else {
    __pyx_t_3 = __pyx_t_2;
    goto __pyx_L16_bool_binop_done;
  }
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_load_aux_arr) != Py_None);
  __pyx_t_3 = __pyx_t_2;
  __pyx_L16_bool_binop_done:;
  if (__pyx_t_3) {

    /* "cupy/fft/_callback.pyx":819
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")
 *         if cb_load_data is None and cb_load_aux_arr is not None:
 *             cb_load_data = cb_load_aux_arr.data             # <<<<<<<<<<<<<<
 *         if cb_store_data is None and cb_store_aux_arr is not None:
 *             cb_store_data = cb_store_aux_arr.data
*/
    __pyx_t_4 = ((PyObject *)__pyx_v_cb_load_aux_arr->data);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_DECREF_SET(__pyx_v_cb_load_data, ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "cupy/fft/_callback.pyx":818
 *         if cb_store_aux_arr is not None and cb_store_data is not None:
 *             raise ValueError("only use cb_store_data and not cb_store_aux_arr")
 *         if cb_load_data is None and cb_load_aux_arr is not None:             # <<<<<<<<<<<<<<
 *             cb_load_data = cb_load_aux_arr.data
 *         if cb_store_data is None and cb_store_aux_arr is not None:
*/
  }

  /* "cupy/fft/_callback.pyx":820
 *         if cb_load_data is None and cb_load_aux_arr is not None:
 *             cb_load_data = cb_load_aux_arr.data
 *         if cb_store_data is None and cb_store_aux_arr is not None:             # <<<<<<<<<<<<<<
 *             cb_store_data = cb_store_aux_arr.data
 * 
*/
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_store_data) == Py_None);
  if (__pyx_t_2) {
  } else {
    __pyx_t_3 = __pyx_t_2;
    goto __pyx_L19_bool_binop_done;
  }
  __pyx_t_2 = (((PyObject *)__pyx_v_cb_store_aux_arr) != Py_None);
  __pyx_t_3 = __pyx_t_2;
  __pyx_L19_bool_binop_done:;
  if (__pyx_t_3) {

    /* "cupy/fft/_callback.pyx":821
 *             cb_load_data = cb_load_aux_arr.data
 *         if cb_store_data is None and cb_store_aux_arr is not None:
 *             cb_store_data = cb_store_aux_arr.data             # <<<<<<<<<<<<<<
 * 
 *         # For every distinct pair of load & store callbacks, we compile an
*/
    __pyx_t_4 = ((PyObject *)__pyx_v_cb_store_aux_arr->data);
    __Pyx_INCREF(__pyx_t_4);
    __Pyx_DECREF_SET(__pyx_v_cb_store_data, ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "cupy/fft/_callback.pyx":820
 *         if cb_load_data is None and cb_load_aux_arr is not None:
 *             cb_load_data = cb_load_aux_arr.data
 *         if cb_store_data is None and cb_store_aux_arr is not None:             # <<<<<<<<<<<<<<
 *             cb_store_data = cb_store_aux_arr.data
 * 
*/
  }

  /* "cupy/fft/_callback.pyx":825
 *         # For every distinct pair of load & store callbacks, we compile an
 *         # external Python module (legacy) or LTO IR (jit) and cache it.
 *         cdef tuple key = (cb_load, cb_store, cb_ver)             # <<<<<<<<<<<<<<
 *         cdef _CallbackManager mgr = _callback_mgr.get(key)
 *         if mgr is None:
*/
  __pyx_t_4 = PyTuple_New(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 825, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_INCREF(__pyx_v_cb_load);
  __Pyx_GIVEREF(__pyx_v_cb_load);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_cb_load) != (0)) __PYX_ERR(0, 825, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_cb_store);
  __Pyx_GIVEREF(__pyx_v_cb_store);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_cb_store) != (0)) __PYX_ERR(0, 825, __pyx_L1_error);
  __Pyx_INCREF(__pyx_v_cb_ver);
  __Pyx_GIVEREF(__pyx_v_cb_ver);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_v_cb_ver) != (0)) __PYX_ERR(0, 825, __pyx_L1_error);
  __pyx_v_key = ((PyObject*)__pyx_t_4);
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":826
 *         # external Python module (legacy) or LTO IR (jit) and cache it.
 *         cdef tuple key = (cb_load, cb_store, cb_ver)
 *         cdef _CallbackManager mgr = _callback_mgr.get(key)             # <<<<<<<<<<<<<<
 *         if mgr is None:
 *             if cb_ver == 'legacy':
*/
  if (unlikely(__pyx_v_4cupy_3fft_9_callback__callback_mgr == Py_None)) {
    PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "get");
    __PYX_ERR(0, 826, __pyx_L1_error)
  }
  __pyx_t_4 = __Pyx_PyDict_GetItemDefault(__pyx_v_4cupy_3fft_9_callback__callback_mgr, __pyx_v_key, Py_None); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 826, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager))))) __PYX_ERR(0, 826, __pyx_L1_error)
  __pyx_v_mgr = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_4);
  __pyx_t_4 = 0;

  /* "cupy/fft/_callback.pyx":827
 *         cdef tuple key = (cb_load, cb_store, cb_ver)
 *         cdef _CallbackManager mgr = _callback_mgr.get(key)
 *         if mgr is None:             # <<<<<<<<<<<<<<
 *             if cb_ver == 'legacy':
 *                 warnings.warn(
*/
  __pyx_t_3 = (((PyObject *)__pyx_v_mgr) == Py_None);
  if (__pyx_t_3) {

    /* "cupy/fft/_callback.pyx":828
 *         cdef _CallbackManager mgr = _callback_mgr.get(key)
 *         if mgr is None:
 *             if cb_ver == 'legacy':             # <<<<<<<<<<<<<<
 *                 warnings.warn(
 *                     'Using the legacy callback is considered deprecated, '
*/
    __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_cb_ver, __pyx_mstate_global->__pyx_n_u_legacy, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 828, __pyx_L1_error)
    if (__pyx_t_3) {

      /* "cupy/fft/_callback.pyx":829
 *         if mgr is None:
 *             if cb_ver == 'legacy':
 *                 warnings.warn(             # <<<<<<<<<<<<<<
 *                     'Using the legacy callback is considered deprecated, '
 *                     'please switch to use cb_ver="jit" instead',
*/
      __pyx_t_8 = NULL;
      __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_warnings); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 829, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_5);
      __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_warn); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 829, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_7);
      __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;

      /* "cupy/fft/_callback.pyx":832
 *                     'Using the legacy callback is considered deprecated, '
 *                     'please switch to use cb_ver="jit" instead',
 *                     DeprecationWarning)             # <<<<<<<<<<<<<<
 *                 mgr = _LegacyCallbackManager(
 *                     cb_load='' if cb_load is None else cb_load,
*/
      __pyx_t_6 = 1;
      #if CYTHON_UNPACK_METHODS
      if (unlikely(PyMethod_Check(__pyx_t_7))) {
        __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_7);
        assert(__pyx_t_8);
        PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7);
        __Pyx_INCREF(__pyx_t_8);
        __Pyx_INCREF(__pyx__function);
        __Pyx_DECREF_SET(__pyx_t_7, __pyx__function);
        __pyx_t_6 = 0;
      }
      #endif
      {
        PyObject *__pyx_callargs[3] = {__pyx_t_8, __pyx_mstate_global->__pyx_kp_u_Using_the_legacy_callback_is_con, ((PyObject *)(((PyTypeObject*)PyExc_DeprecationWarning)))};
        __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
        __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
        __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
        if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 829, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_4);
      }
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;

      /* "cupy/fft/_callback.pyx":833
 *                     'please switch to use cb_ver="jit" instead',
 *                     DeprecationWarning)
 *                 mgr = _LegacyCallbackManager(             # <<<<<<<<<<<<<<
 *                     cb_load='' if cb_load is None else cb_load,
 *                     cb_store='' if cb_store is None else cb_store,
*/
      __pyx_t_7 = NULL;

      /* "cupy/fft/_callback.pyx":834
 *                     DeprecationWarning)
 *                 mgr = _LegacyCallbackManager(
 *                     cb_load='' if cb_load is None else cb_load,             # <<<<<<<<<<<<<<
 *                     cb_store='' if cb_store is None else cb_store,
 *                     cb_load_data=cb_load_data,
*/
      __pyx_t_3 = (__pyx_v_cb_load == Py_None);
      if (__pyx_t_3) {
        __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u__6);
        __pyx_t_8 = __pyx_mstate_global->__pyx_kp_u__6;
      } else {
        __Pyx_INCREF(__pyx_v_cb_load);
        __pyx_t_8 = __pyx_v_cb_load;
      }

      /* "cupy/fft/_callback.pyx":835
 *                 mgr = _LegacyCallbackManager(
 *                     cb_load='' if cb_load is None else cb_load,
 *                     cb_store='' if cb_store is None else cb_store,             # <<<<<<<<<<<<<<
 *                     cb_load_data=cb_load_data,
 *                     cb_store_data=cb_store_data,
*/
      __pyx_t_3 = (__pyx_v_cb_store == Py_None);
      if (__pyx_t_3) {
        __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u__6);
        __pyx_t_5 = __pyx_mstate_global->__pyx_kp_u__6;
      } else {
        __Pyx_INCREF(__pyx_v_cb_store);
        __pyx_t_5 = __pyx_v_cb_store;
      }

      /* "cupy/fft/_callback.pyx":837
 *                     cb_store='' if cb_store is None else cb_store,
 *                     cb_load_data=cb_load_data,
 *                     cb_store_data=cb_store_data,             # <<<<<<<<<<<<<<
 *                 )
 *                 _callback_mgr[key] = mgr  # keep the Python module alive
*/
      __pyx_t_6 = 1;
      {
        PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 4 : 0)] = {__pyx_t_7, NULL};
        __pyx_t_9 = __Pyx_MakeVectorcallBuilderKwds(4); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 833, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_9);
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_load, __pyx_t_8, __pyx_t_9, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 833, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_store, __pyx_t_5, __pyx_t_9, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 833, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_load_data, ((PyObject *)__pyx_v_cb_load_data), __pyx_t_9, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 833, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_store_data, ((PyObject *)__pyx_v_cb_store_data), __pyx_t_9, __pyx_callargs+1, 3) < (0)) __PYX_ERR(0, 833, __pyx_L1_error)
        __pyx_t_4 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_9);
        __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
        __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
        __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
        __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
        if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 833, __pyx_L1_error)
        __Pyx_GOTREF((PyObject *)__pyx_t_4);
      }
      __Pyx_DECREF_SET(__pyx_v_mgr, ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_4));
      __pyx_t_4 = 0;

      /* "cupy/fft/_callback.pyx":839
 *                     cb_store_data=cb_store_data,
 *                 )
 *                 _callback_mgr[key] = mgr  # keep the Python module alive             # <<<<<<<<<<<<<<
 *             else:  # cb_ver = 'jit'
 *                 # We help users reuse their legacy callbacks with minimal
*/
      if (unlikely(__pyx_v_4cupy_3fft_9_callback__callback_mgr == Py_None)) {
        PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable");
        __PYX_ERR(0, 839, __pyx_L1_error)
      }
      if (unlikely((PyDict_SetItem(__pyx_v_4cupy_3fft_9_callback__callback_mgr, __pyx_v_key, ((PyObject *)__pyx_v_mgr)) < 0))) __PYX_ERR(0, 839, __pyx_L1_error)

      /* "cupy/fft/_callback.pyx":828
 *         cdef _CallbackManager mgr = _callback_mgr.get(key)
 *         if mgr is None:
 *             if cb_ver == 'legacy':             # <<<<<<<<<<<<<<
 *                 warnings.warn(
 *                     'Using the legacy callback is considered deprecated, '
*/
      goto __pyx_L22;
    }

    /* "cupy/fft/_callback.pyx":843
 *                 # We help users reuse their legacy callbacks with minimal
 *                 # code changes
 *                 if isinstance(cb_load, str) and cb_load_name is None:             # <<<<<<<<<<<<<<
 *                     try:
 *                         cb_load_name = cupy_load_callback_pattern.search(
*/
    /*else*/ {
      __pyx_t_2 = PyUnicode_Check(__pyx_v_cb_load); 
      if (__pyx_t_2) {
      } else {
        __pyx_t_3 = __pyx_t_2;
        goto __pyx_L24_bool_binop_done;
      }
      __pyx_t_2 = (__pyx_v_cb_load_name == ((PyObject*)Py_None));
      __pyx_t_3 = __pyx_t_2;
      __pyx_L24_bool_binop_done:;
      if (__pyx_t_3) {

        /* "cupy/fft/_callback.pyx":844
 *                 # code changes
 *                 if isinstance(cb_load, str) and cb_load_name is None:
 *                     try:             # <<<<<<<<<<<<<<
 *                         cb_load_name = cupy_load_callback_pattern.search(
 *                             cb_load).group(1)
*/
        {
          __Pyx_PyThreadState_declare
          __Pyx_PyThreadState_assign
          __Pyx_ExceptionSave(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12);
          __Pyx_XGOTREF(__pyx_t_10);
          __Pyx_XGOTREF(__pyx_t_11);
          __Pyx_XGOTREF(__pyx_t_12);
          /*try:*/ {

            /* "cupy/fft/_callback.pyx":845
 *                 if isinstance(cb_load, str) and cb_load_name is None:
 *                     try:
 *                         cb_load_name = cupy_load_callback_pattern.search(             # <<<<<<<<<<<<<<
 *                             cb_load).group(1)
 *                     except AttributeError:
*/
            __pyx_t_8 = __pyx_v_4cupy_3fft_9_callback_cupy_load_callback_pattern;
            __Pyx_INCREF(__pyx_t_8);

            /* "cupy/fft/_callback.pyx":846
 *                     try:
 *                         cb_load_name = cupy_load_callback_pattern.search(
 *                             cb_load).group(1)             # <<<<<<<<<<<<<<
 *                     except AttributeError:
 *                         raise ValueError(
*/
            __pyx_t_6 = 0;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_v_cb_load};
              __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_search, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
              if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 845, __pyx_L26_error)
              __Pyx_GOTREF(__pyx_t_5);
            }
            __pyx_t_9 = __pyx_t_5;
            __Pyx_INCREF(__pyx_t_9);
            __pyx_t_6 = 0;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_int_1};
              __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_group, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0;
              __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
              if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 846, __pyx_L26_error)
              __Pyx_GOTREF(__pyx_t_4);
            }
            if (!(likely(PyUnicode_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_4))) __PYX_ERR(0, 846, __pyx_L26_error)
            __Pyx_DECREF_SET(__pyx_v_cb_load_name, ((PyObject*)__pyx_t_4));
            __pyx_t_4 = 0;

            /* "cupy/fft/_callback.pyx":844
 *                 # code changes
 *                 if isinstance(cb_load, str) and cb_load_name is None:
 *                     try:             # <<<<<<<<<<<<<<
 *                         cb_load_name = cupy_load_callback_pattern.search(
 *                             cb_load).group(1)
*/
          }
          __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
          __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0;
          __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0;
          goto __pyx_L31_try_end;
          __pyx_L26_error:;
          __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
          __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
          __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
          __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
          __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
          __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0;

          /* "cupy/fft/_callback.pyx":847
 *                         cb_load_name = cupy_load_callback_pattern.search(
 *                             cb_load).group(1)
 *                     except AttributeError:             # <<<<<<<<<<<<<<
 *                         raise ValueError(
 *                             'cb_load_name cannot be inferred, please specify '
*/
          __pyx_t_13 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_AttributeError))));
          if (__pyx_t_13) {
            __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
            if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_5, &__pyx_t_9) < 0) __PYX_ERR(0, 847, __pyx_L28_except_error)
            __Pyx_XGOTREF(__pyx_t_4);
            __Pyx_XGOTREF(__pyx_t_5);
            __Pyx_XGOTREF(__pyx_t_9);

            /* "cupy/fft/_callback.pyx":848
 *                             cb_load).group(1)
 *                     except AttributeError:
 *                         raise ValueError(             # <<<<<<<<<<<<<<
 *                             'cb_load_name cannot be inferred, please specify '
 *                             'it explicitly') from None
*/
            __pyx_t_7 = NULL;
            __pyx_t_6 = 1;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_mstate_global->__pyx_kp_u_cb_load_name_cannot_be_inferred};
              __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
              if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 848, __pyx_L28_except_error)
              __Pyx_GOTREF(__pyx_t_8);
            }

            /* "cupy/fft/_callback.pyx":850
 *                         raise ValueError(
 *                             'cb_load_name cannot be inferred, please specify '
 *                             'it explicitly') from None             # <<<<<<<<<<<<<<
 *                 if isinstance(cb_store, str) and cb_store_name is None:
 *                     try:
*/
            __Pyx_Raise(__pyx_t_8, 0, 0, Py_None);
            __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
            __PYX_ERR(0, 848, __pyx_L28_except_error)
          }
          goto __pyx_L28_except_error;

          /* "cupy/fft/_callback.pyx":844
 *                 # code changes
 *                 if isinstance(cb_load, str) and cb_load_name is None:
 *                     try:             # <<<<<<<<<<<<<<
 *                         cb_load_name = cupy_load_callback_pattern.search(
 *                             cb_load).group(1)
*/
          __pyx_L28_except_error:;
          __Pyx_XGIVEREF(__pyx_t_10);
          __Pyx_XGIVEREF(__pyx_t_11);
          __Pyx_XGIVEREF(__pyx_t_12);
          __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12);
          goto __pyx_L1_error;
          __pyx_L31_try_end:;
        }

        /* "cupy/fft/_callback.pyx":843
 *                 # We help users reuse their legacy callbacks with minimal
 *                 # code changes
 *                 if isinstance(cb_load, str) and cb_load_name is None:             # <<<<<<<<<<<<<<
 *                     try:
 *                         cb_load_name = cupy_load_callback_pattern.search(
*/
      }

      /* "cupy/fft/_callback.pyx":851
 *                             'cb_load_name cannot be inferred, please specify '
 *                             'it explicitly') from None
 *                 if isinstance(cb_store, str) and cb_store_name is None:             # <<<<<<<<<<<<<<
 *                     try:
 *                         cb_store_name = cupy_store_callback_pattern.search(
*/
      __pyx_t_2 = PyUnicode_Check(__pyx_v_cb_store); 
      if (__pyx_t_2) {
      } else {
        __pyx_t_3 = __pyx_t_2;
        goto __pyx_L35_bool_binop_done;
      }
      __pyx_t_2 = (__pyx_v_cb_store_name == ((PyObject*)Py_None));
      __pyx_t_3 = __pyx_t_2;
      __pyx_L35_bool_binop_done:;
      if (__pyx_t_3) {

        /* "cupy/fft/_callback.pyx":852
 *                             'it explicitly') from None
 *                 if isinstance(cb_store, str) and cb_store_name is None:
 *                     try:             # <<<<<<<<<<<<<<
 *                         cb_store_name = cupy_store_callback_pattern.search(
 *                             cb_store).group(1)
*/
        {
          __Pyx_PyThreadState_declare
          __Pyx_PyThreadState_assign
          __Pyx_ExceptionSave(&__pyx_t_12, &__pyx_t_11, &__pyx_t_10);
          __Pyx_XGOTREF(__pyx_t_12);
          __Pyx_XGOTREF(__pyx_t_11);
          __Pyx_XGOTREF(__pyx_t_10);
          /*try:*/ {

            /* "cupy/fft/_callback.pyx":853
 *                 if isinstance(cb_store, str) and cb_store_name is None:
 *                     try:
 *                         cb_store_name = cupy_store_callback_pattern.search(             # <<<<<<<<<<<<<<
 *                             cb_store).group(1)
 *                     except AttributeError:
*/
            __pyx_t_8 = __pyx_v_4cupy_3fft_9_callback_cupy_store_callback_pattern;
            __Pyx_INCREF(__pyx_t_8);

            /* "cupy/fft/_callback.pyx":854
 *                     try:
 *                         cb_store_name = cupy_store_callback_pattern.search(
 *                             cb_store).group(1)             # <<<<<<<<<<<<<<
 *                     except AttributeError:
 *                         raise ValueError(
*/
            __pyx_t_6 = 0;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_v_cb_store};
              __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_search, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
              if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 853, __pyx_L37_error)
              __Pyx_GOTREF(__pyx_t_4);
            }
            __pyx_t_5 = __pyx_t_4;
            __Pyx_INCREF(__pyx_t_5);
            __pyx_t_6 = 0;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_int_1};
              __pyx_t_9 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_group, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
              __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
              if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 854, __pyx_L37_error)
              __Pyx_GOTREF(__pyx_t_9);
            }
            if (!(likely(PyUnicode_CheckExact(__pyx_t_9))||((__pyx_t_9) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_9))) __PYX_ERR(0, 854, __pyx_L37_error)
            __Pyx_DECREF_SET(__pyx_v_cb_store_name, ((PyObject*)__pyx_t_9));
            __pyx_t_9 = 0;

            /* "cupy/fft/_callback.pyx":852
 *                             'it explicitly') from None
 *                 if isinstance(cb_store, str) and cb_store_name is None:
 *                     try:             # <<<<<<<<<<<<<<
 *                         cb_store_name = cupy_store_callback_pattern.search(
 *                             cb_store).group(1)
*/
          }
          __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0;
          __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0;
          __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0;
          goto __pyx_L42_try_end;
          __pyx_L37_error:;
          __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
          __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
          __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
          __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
          __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
          __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0;

          /* "cupy/fft/_callback.pyx":855
 *                         cb_store_name = cupy_store_callback_pattern.search(
 *                             cb_store).group(1)
 *                     except AttributeError:             # <<<<<<<<<<<<<<
 *                         raise ValueError(
 *                             'cb_store_name cannot be inferred, please specify '
*/
          __pyx_t_13 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_AttributeError))));
          if (__pyx_t_13) {
            __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
            if (__Pyx_GetException(&__pyx_t_9, &__pyx_t_4, &__pyx_t_5) < 0) __PYX_ERR(0, 855, __pyx_L39_except_error)
            __Pyx_XGOTREF(__pyx_t_9);
            __Pyx_XGOTREF(__pyx_t_4);
            __Pyx_XGOTREF(__pyx_t_5);

            /* "cupy/fft/_callback.pyx":856
 *                             cb_store).group(1)
 *                     except AttributeError:
 *                         raise ValueError(             # <<<<<<<<<<<<<<
 *                             'cb_store_name cannot be inferred, please specify '
 *                             'it explicitly') from None
*/
            __pyx_t_7 = NULL;
            __pyx_t_6 = 1;
            {
              PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_mstate_global->__pyx_kp_u_cb_store_name_cannot_be_inferred};
              __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
              __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
              if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 856, __pyx_L39_except_error)
              __Pyx_GOTREF(__pyx_t_8);
            }

            /* "cupy/fft/_callback.pyx":858
 *                         raise ValueError(
 *                             'cb_store_name cannot be inferred, please specify '
 *                             'it explicitly') from None             # <<<<<<<<<<<<<<
 *                 mgr = _JITCallbackManager(
 *                     cb_load=cb_load,
*/
            __Pyx_Raise(__pyx_t_8, 0, 0, Py_None);
            __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
            __PYX_ERR(0, 856, __pyx_L39_except_error)
          }
          goto __pyx_L39_except_error;

          /* "cupy/fft/_callback.pyx":852
 *                             'it explicitly') from None
 *                 if isinstance(cb_store, str) and cb_store_name is None:
 *                     try:             # <<<<<<<<<<<<<<
 *                         cb_store_name = cupy_store_callback_pattern.search(
 *                             cb_store).group(1)
*/
          __pyx_L39_except_error:;
          __Pyx_XGIVEREF(__pyx_t_12);
          __Pyx_XGIVEREF(__pyx_t_11);
          __Pyx_XGIVEREF(__pyx_t_10);
          __Pyx_ExceptionReset(__pyx_t_12, __pyx_t_11, __pyx_t_10);
          goto __pyx_L1_error;
          __pyx_L42_try_end:;
        }

        /* "cupy/fft/_callback.pyx":851
 *                             'cb_load_name cannot be inferred, please specify '
 *                             'it explicitly') from None
 *                 if isinstance(cb_store, str) and cb_store_name is None:             # <<<<<<<<<<<<<<
 *                     try:
 *                         cb_store_name = cupy_store_callback_pattern.search(
*/
      }

      /* "cupy/fft/_callback.pyx":859
 *                             'cb_store_name cannot be inferred, please specify '
 *                             'it explicitly') from None
 *                 mgr = _JITCallbackManager(             # <<<<<<<<<<<<<<
 *                     cb_load=cb_load,
 *                     cb_store=cb_store,
*/
      __pyx_t_4 = NULL;

      /* "cupy/fft/_callback.pyx":866
 *                     cb_load_data=cb_load_data,
 *                     cb_store_data=cb_store_data,
 *                     nvrtc_options=nvrtc_options,             # <<<<<<<<<<<<<<
 *                 )
 *         else:
*/
      __pyx_t_6 = 1;
      {
        PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 7 : 0)] = {__pyx_t_4, NULL};
        __pyx_t_9 = __Pyx_MakeVectorcallBuilderKwds(7); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 859, __pyx_L1_error)
        __Pyx_GOTREF(__pyx_t_9);
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_load, __pyx_v_cb_load, __pyx_t_9, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_store, __pyx_v_cb_store, __pyx_t_9, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_load_name, __pyx_v_cb_load_name, __pyx_t_9, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_store_name, __pyx_v_cb_store_name, __pyx_t_9, __pyx_callargs+1, 3) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_load_data, ((PyObject *)__pyx_v_cb_load_data), __pyx_t_9, __pyx_callargs+1, 4) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_cb_store_data, ((PyObject *)__pyx_v_cb_store_data), __pyx_t_9, __pyx_callargs+1, 5) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_nvrtc_options, __pyx_v_nvrtc_options, __pyx_t_9, __pyx_callargs+1, 6) < (0)) __PYX_ERR(0, 859, __pyx_L1_error)
        __pyx_t_5 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_9);
        __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
        __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
        if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 859, __pyx_L1_error)
        __Pyx_GOTREF((PyObject *)__pyx_t_5);
      }
      __Pyx_DECREF_SET(__pyx_v_mgr, ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_5));
      __pyx_t_5 = 0;
    }
    __pyx_L22:;

    /* "cupy/fft/_callback.pyx":827
 *         cdef tuple key = (cb_load, cb_store, cb_ver)
 *         cdef _CallbackManager mgr = _callback_mgr.get(key)
 *         if mgr is None:             # <<<<<<<<<<<<<<
 *             if cb_ver == 'legacy':
 *                 warnings.warn(
*/
    goto __pyx_L21;
  }

  /* "cupy/fft/_callback.pyx":869
 *                 )
 *         else:
 *             assert mgr.identity == cb_ver             # <<<<<<<<<<<<<<
 *             mgr.set_caller_infos(
 *                 cb_load_data=cb_load_data,
*/
  /*else*/ {
    #ifndef CYTHON_WITHOUT_ASSERTIONS
    if (unlikely(__pyx_assertions_enabled())) {
      __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mgr->identity, __pyx_v_cb_ver, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 869, __pyx_L1_error)
      if (unlikely(!__pyx_t_3)) {
        __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0);
        __PYX_ERR(0, 869, __pyx_L1_error)
      }
    }
    #else
    if ((1)); else __PYX_ERR(0, 869, __pyx_L1_error)
    #endif

    /* "cupy/fft/_callback.pyx":870
 *         else:
 *             assert mgr.identity == cb_ver
 *             mgr.set_caller_infos(             # <<<<<<<<<<<<<<
 *                 cb_load_data=cb_load_data,
 *                 cb_store_data=cb_store_data)
*/
    __pyx_t_14.__pyx_n = 2;
    __pyx_t_14.cb_load_data = __pyx_v_cb_load_data;
    __pyx_t_14.cb_store_data = __pyx_v_cb_store_data;
    __pyx_t_5 = ((struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager *)__pyx_v_mgr->__pyx_vtab)->set_caller_infos(__pyx_v_mgr, &__pyx_t_14); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 870, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
  }
  __pyx_L21:;

  /* "cupy/fft/_callback.pyx":873
 *                 cb_load_data=cb_load_data,
 *                 cb_store_data=cb_store_data)
 *         self.mgr = mgr             # <<<<<<<<<<<<<<
 * 
 *     def __enter__(self):
*/
  __Pyx_INCREF((PyObject *)__pyx_v_mgr);
  __Pyx_GIVEREF((PyObject *)__pyx_v_mgr);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->mgr);
  __Pyx_DECREF((PyObject *)__pyx_v_self->mgr);
  __pyx_v_self->mgr = __pyx_v_mgr;

  /* "cupy/fft/_callback.pyx":794
 *         _CallbackManager mgr_prev
 * 
 *     def __init__(self,             # <<<<<<<<<<<<<<
 *                  cb_load=None,
 *                  cb_store=None,
*/

  /* function exit code */
  __pyx_r = 0;
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_7);
  __Pyx_XDECREF(__pyx_t_8);
  __Pyx_XDECREF(__pyx_t_9);
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = -1;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_key);
  __Pyx_XDECREF((PyObject *)__pyx_v_mgr);
  __Pyx_XDECREF(__pyx_v_cb_load_name);
  __Pyx_XDECREF(__pyx_v_cb_store_name);
  __Pyx_XDECREF((PyObject *)__pyx_v_cb_load_data);
  __Pyx_XDECREF((PyObject *)__pyx_v_cb_store_data);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":875
 *         self.mgr = mgr
 * 
 *     def __enter__(self):             # <<<<<<<<<<<<<<
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         self.mgr_prev = tls._current_cufft_callback
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_3__enter__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_2__enter__, "set_cufft_callbacks.__enter__(self)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_3__enter__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__enter__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__enter__", 1, 0, 0, __pyx_nargs); return NULL; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return NULL;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__enter__", __pyx_kwds); return NULL;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_2__enter__(((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_2__enter__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self) {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_tls = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__enter__", 0);

  /* "cupy/fft/_callback.pyx":876
 * 
 *     def __enter__(self):
 *         cdef _ThreadLocal tls = _ThreadLocal.get()             # <<<<<<<<<<<<<<
 *         self.mgr_prev = tls._current_cufft_callback
 *         tls._current_cufft_callback = self.mgr
*/
  __pyx_t_1 = ((PyObject *)__pyx_f_4cupy_3fft_9_callback_12_ThreadLocal_get()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 876, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_tls = ((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":877
 *     def __enter__(self):
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         self.mgr_prev = tls._current_cufft_callback             # <<<<<<<<<<<<<<
 *         tls._current_cufft_callback = self.mgr
 *         return self.mgr
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_tls->_current_cufft_callback);
  __Pyx_INCREF(__pyx_t_1);
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v_self->mgr_prev);
  __Pyx_DECREF((PyObject *)__pyx_v_self->mgr_prev);
  __pyx_v_self->mgr_prev = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":878
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         self.mgr_prev = tls._current_cufft_callback
 *         tls._current_cufft_callback = self.mgr             # <<<<<<<<<<<<<<
 *         return self.mgr
 * 
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_self->mgr);
  __Pyx_INCREF(__pyx_t_1);
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v_tls->_current_cufft_callback);
  __Pyx_DECREF((PyObject *)__pyx_v_tls->_current_cufft_callback);
  __pyx_v_tls->_current_cufft_callback = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":879
 *         self.mgr_prev = tls._current_cufft_callback
 *         tls._current_cufft_callback = self.mgr
 *         return self.mgr             # <<<<<<<<<<<<<<
 * 
 *     def __exit__(self, exc_type, exc_value, traceback):
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF((PyObject *)__pyx_v_self->mgr);
  __pyx_r = ((PyObject *)__pyx_v_self->mgr);
  goto __pyx_L0;

  /* "cupy/fft/_callback.pyx":875
 *         self.mgr = mgr
 * 
 *     def __enter__(self):             # <<<<<<<<<<<<<<
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         self.mgr_prev = tls._current_cufft_callback
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__enter__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF((PyObject *)__pyx_v_tls);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "cupy/fft/_callback.pyx":881
 *         return self.mgr
 * 
 *     def __exit__(self, exc_type, exc_value, traceback):             # <<<<<<<<<<<<<<
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         tls._current_cufft_callback = self.mgr_prev
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_5__exit__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_4__exit__, "set_cufft_callbacks.__exit__(self, exc_type, exc_value, traceback)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_5__exit__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  CYTHON_UNUSED PyObject *__pyx_v_exc_type = 0;
  CYTHON_UNUSED PyObject *__pyx_v_exc_value = 0;
  CYTHON_UNUSED PyObject *__pyx_v_traceback = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[3] = {0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__exit__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_exc_type,&__pyx_mstate_global->__pyx_n_u_exc_value,&__pyx_mstate_global->__pyx_n_u_traceback,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(0, 881, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  3:
        values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 881, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 881, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 881, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__exit__", 0) < (0)) __PYX_ERR(0, 881, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 3; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__exit__", 1, 3, 3, i); __PYX_ERR(0, 881, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 3)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 881, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 881, __pyx_L3_error)
      values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 881, __pyx_L3_error)
    }
    __pyx_v_exc_type = values[0];
    __pyx_v_exc_value = values[1];
    __pyx_v_traceback = values[2];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__exit__", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 881, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__exit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_4__exit__(((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)__pyx_v_self), __pyx_v_exc_type, __pyx_v_exc_value, __pyx_v_traceback);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_4__exit__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_exc_type, CYTHON_UNUSED PyObject *__pyx_v_exc_value, CYTHON_UNUSED PyObject *__pyx_v_traceback) {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v_tls = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__exit__", 0);

  /* "cupy/fft/_callback.pyx":882
 * 
 *     def __exit__(self, exc_type, exc_value, traceback):
 *         cdef _ThreadLocal tls = _ThreadLocal.get()             # <<<<<<<<<<<<<<
 *         tls._current_cufft_callback = self.mgr_prev
 *         # do not remove mgr from _callback_mgr, as one might still wanna use
*/
  __pyx_t_1 = ((PyObject *)__pyx_f_4cupy_3fft_9_callback_12_ThreadLocal_get()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 882, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v_tls = ((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":883
 *     def __exit__(self, exc_type, exc_value, traceback):
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         tls._current_cufft_callback = self.mgr_prev             # <<<<<<<<<<<<<<
 *         # do not remove mgr from _callback_mgr, as one might still wanna use
 *         # plans generated by it in the same Python session
*/
  __pyx_t_1 = ((PyObject *)__pyx_v_self->mgr_prev);
  __Pyx_INCREF(__pyx_t_1);
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v_tls->_current_cufft_callback);
  __Pyx_DECREF((PyObject *)__pyx_v_tls->_current_cufft_callback);
  __pyx_v_tls->_current_cufft_callback = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "cupy/fft/_callback.pyx":881
 *         return self.mgr
 * 
 *     def __exit__(self, exc_type, exc_value, traceback):             # <<<<<<<<<<<<<<
 *         cdef _ThreadLocal tls = _ThreadLocal.get()
 *         tls._current_cufft_callback = self.mgr_prev
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__exit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF((PyObject *)__pyx_v_tls);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_7__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_6__reduce_cython__, "set_cufft_callbacks.__reduce_cython__(self)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_7__reduce_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; }
  const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
  if (unlikely(__pyx_kwds_len < 0)) return NULL;
  if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;}
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_6__reduce_cython__(((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)__pyx_v_self));

  /* function exit code */
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_6__reduce_cython__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self) {
  PyObject *__pyx_v_state = 0;
  PyObject *__pyx_v__dict = 0;
  int __pyx_v_use_setstate;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__reduce_cython__", 0);

  /* "(tree fragment)":5
 *     cdef object _dict
 *     cdef bint use_setstate
 *     state = (self.mgr, self.mgr_prev)             # <<<<<<<<<<<<<<
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
*/
  __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_INCREF((PyObject *)__pyx_v_self->mgr);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->mgr);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self->mgr)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __Pyx_INCREF((PyObject *)__pyx_v_self->mgr_prev);
  __Pyx_GIVEREF((PyObject *)__pyx_v_self->mgr_prev);
  if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_self->mgr_prev)) != (0)) __PYX_ERR(3, 5, __pyx_L1_error);
  __pyx_v_state = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":6
 *     cdef bint use_setstate
 *     state = (self.mgr, self.mgr_prev)
 *     _dict = getattr(self, '__dict__', None)             # <<<<<<<<<<<<<<
 *     if _dict is not None and _dict:
 *         state += (_dict,)
*/
  __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 6, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_v__dict = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "(tree fragment)":7
 *     state = (self.mgr, self.mgr_prev)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
  __pyx_t_3 = (__pyx_v__dict != Py_None);
  if (__pyx_t_3) {
  } else {
    __pyx_t_2 = __pyx_t_3;
    goto __pyx_L4_bool_binop_done;
  }
  __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(3, 7, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_3;
  __pyx_L4_bool_binop_done:;
  if (__pyx_t_2) {

    /* "(tree fragment)":8
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:
 *         state += (_dict,)             # <<<<<<<<<<<<<<
 *         use_setstate = True
 *     else:
*/
    __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(__pyx_v__dict);
    __Pyx_GIVEREF(__pyx_v__dict);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(3, 8, __pyx_L1_error);
    __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 8, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
    __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4));
    __pyx_t_4 = 0;

    /* "(tree fragment)":9
 *     if _dict is not None and _dict:
 *         state += (_dict,)
 *         use_setstate = True             # <<<<<<<<<<<<<<
 *     else:
 *         use_setstate = self.mgr is not None or self.mgr_prev is not None
*/
    __pyx_v_use_setstate = 1;

    /* "(tree fragment)":7
 *     state = (self.mgr, self.mgr_prev)
 *     _dict = getattr(self, '__dict__', None)
 *     if _dict is not None and _dict:             # <<<<<<<<<<<<<<
 *         state += (_dict,)
 *         use_setstate = True
*/
    goto __pyx_L3;
  }

  /* "(tree fragment)":11
 *         use_setstate = True
 *     else:
 *         use_setstate = self.mgr is not None or self.mgr_prev is not None             # <<<<<<<<<<<<<<
 *     if use_setstate:
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, None), state
*/
  /*else*/ {
    __pyx_t_3 = (((PyObject *)__pyx_v_self->mgr) != Py_None);
    if (!__pyx_t_3) {
    } else {
      __pyx_t_2 = __pyx_t_3;
      goto __pyx_L6_bool_binop_done;
    }
    __pyx_t_3 = (((PyObject *)__pyx_v_self->mgr_prev) != Py_None);
    __pyx_t_2 = __pyx_t_3;
    __pyx_L6_bool_binop_done:;
    __pyx_v_use_setstate = __pyx_t_2;
  }
  __pyx_L3:;

  /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.mgr is not None or self.mgr_prev is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, None), state
 *     else:
*/
  if (__pyx_v_use_setstate) {

    /* "(tree fragment)":13
 *         use_setstate = self.mgr is not None or self.mgr_prev is not None
 *     if use_setstate:
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, None), state             # <<<<<<<<<<<<<<
 *     else:
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, state)
*/
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_set_cufft_callbac); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_258762224);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_258762224);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_258762224) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(Py_None);
    __Pyx_GIVEREF(Py_None);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 13, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __Pyx_GIVEREF(__pyx_t_4);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 13, __pyx_L1_error);
    __pyx_t_4 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_5;
    __pyx_t_5 = 0;
    goto __pyx_L0;

    /* "(tree fragment)":12
 *     else:
 *         use_setstate = self.mgr is not None or self.mgr_prev is not None
 *     if use_setstate:             # <<<<<<<<<<<<<<
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, None), state
 *     else:
*/
  }

  /* "(tree fragment)":15
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, None), state
 *     else:
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, state)             # <<<<<<<<<<<<<<
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle_set_cufft_callbacks__set_state(self, __pyx_state)
*/
  /*else*/ {
    __Pyx_XDECREF(__pyx_r);
    __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_set_cufft_callbac); if (unlikely(!__pyx_t_5)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_5);
    __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))));
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_mstate_global->__pyx_int_258762224);
    __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_258762224);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_258762224) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_INCREF(__pyx_v_state);
    __Pyx_GIVEREF(__pyx_v_state);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(3, 15, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_4);
    __Pyx_GIVEREF(__pyx_t_5);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __Pyx_GIVEREF(__pyx_t_1);
    if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(3, 15, __pyx_L1_error);
    __pyx_t_5 = 0;
    __pyx_t_1 = 0;
    __pyx_r = __pyx_t_4;
    __pyx_t_4 = 0;
    goto __pyx_L0;
  }

  /* "(tree fragment)":1
 * def __reduce_cython__(self):             # <<<<<<<<<<<<<<
 *     cdef tuple state
 *     cdef object _dict
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v_state);
  __Pyx_XDECREF(__pyx_v__dict);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle_set_cufft_callbacks__set_state(self, __pyx_state)
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_9__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_8__setstate_cython__, "set_cufft_callbacks.__setstate_cython__(self, __pyx_state)");
static PyObject *__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_9__setstate_cython__(PyObject *__pyx_v_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[1] = {0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 16, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(3, 16, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 1; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(3, 16, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 1)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 16, __pyx_L3_error)
    }
    __pyx_v___pyx_state = values[0];
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(3, 16, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_8__setstate_cython__(((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)__pyx_v_self), __pyx_v___pyx_state);

  /* function exit code */
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_19set_cufft_callbacks_8__setstate_cython__(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v_self, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__setstate_cython__", 0);

  /* "(tree fragment)":17
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, state)
 * def __setstate_cython__(self, __pyx_state):
 *     __pyx_unpickle_set_cufft_callbacks__set_state(self, __pyx_state)             # <<<<<<<<<<<<<<
*/
  __pyx_t_1 = __pyx_v___pyx_state;
  __Pyx_INCREF(__pyx_t_1);
  if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(3, 17, __pyx_L1_error)
  if (unlikely(__pyx_t_1 == Py_None)) {
    PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
    __PYX_ERR(3, 17, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle_set_cufft_callbacks__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 17, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":16
 *     else:
 *         return __pyx_unpickle_set_cufft_callbacks, (type(self), 0xf6c65f0, state)
 * def __setstate_cython__(self, __pyx_state):             # <<<<<<<<<<<<<<
 *     __pyx_unpickle_set_cufft_callbacks__set_state(self, __pyx_state)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_AddTraceback("cupy.fft._callback.set_cufft_callbacks.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__ThreadLocal(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_5__pyx_unpickle__ThreadLocal(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_4__pyx_unpickle__ThreadLocal, "__pyx_unpickle__ThreadLocal(__pyx_type, long __pyx_checksum, tuple __pyx_state)");
static PyMethodDef __pyx_mdef_4cupy_3fft_9_callback_5__pyx_unpickle__ThreadLocal = {"__pyx_unpickle__ThreadLocal", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_5__pyx_unpickle__ThreadLocal, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_4__pyx_unpickle__ThreadLocal};
static PyObject *__pyx_pw_4cupy_3fft_9_callback_5__pyx_unpickle__ThreadLocal(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_type = 0;
  long __pyx_v___pyx_checksum;
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[3] = {0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__pyx_unpickle__ThreadLocal (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 4, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  3:
        values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle__ThreadLocal", 0) < (0)) __PYX_ERR(3, 4, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 3; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__ThreadLocal", 1, 3, 3, i); __PYX_ERR(3, 4, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 3)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
    }
    __pyx_v___pyx_type = values[0];
    __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(3, 4, __pyx_L3_error)
    __pyx_v___pyx_state = ((PyObject*)values[2]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__ThreadLocal", 1, 3, 3, __pyx_nargs); __PYX_ERR(3, 4, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__ThreadLocal", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(3, 4, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_4__pyx_unpickle__ThreadLocal(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_4__pyx_unpickle__ThreadLocal(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_v___pyx_result = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  size_t __pyx_t_4;
  int __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__ThreadLocal", 0);

  /* "(tree fragment)":6
 * def __pyx_unpickle__ThreadLocal(__pyx_type, long __pyx_checksum, tuple __pyx_state):
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')             # <<<<<<<<<<<<<<
 *     __pyx_result = _ThreadLocal.__new__(__pyx_type)
 *     if __pyx_state is not None:
*/
  __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, __pyx_k_current_cufft_callback); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(3, 6, __pyx_L1_error)

  /* "(tree fragment)":7
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')
 *     __pyx_result = _ThreadLocal.__new__(__pyx_type)             # <<<<<<<<<<<<<<
 *     if __pyx_state is not None:
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)
*/
  __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal);
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_4 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 7, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_v___pyx_result = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')
 *     __pyx_result = _ThreadLocal.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None));
  if (__pyx_t_5) {

    /* "(tree fragment)":9
 *     __pyx_result = _ThreadLocal.__new__(__pyx_type)
 *     if __pyx_state is not None:
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)             # <<<<<<<<<<<<<<
 *     return __pyx_result
 * cdef __pyx_unpickle__ThreadLocal__set_state(_ThreadLocal __pyx_result, __pyx_state: tuple):
*/
    if (unlikely(__pyx_v___pyx_state == Py_None)) {
      PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
      __PYX_ERR(3, 9, __pyx_L1_error)
    }
    __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__ThreadLocal__set_state(((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 9, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')
 *     __pyx_result = _ThreadLocal.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  }

  /* "(tree fragment)":10
 *     if __pyx_state is not None:
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)
 *     return __pyx_result             # <<<<<<<<<<<<<<
 * cdef __pyx_unpickle__ThreadLocal__set_state(_ThreadLocal __pyx_result, __pyx_state: tuple):
 *     __pyx_result._current_cufft_callback = __pyx_state[0]
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v___pyx_result);
  __pyx_r = __pyx_v___pyx_result;
  goto __pyx_L0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__ThreadLocal(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__ThreadLocal", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v___pyx_result);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":11
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__ThreadLocal__set_state(_ThreadLocal __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result._current_cufft_callback = __pyx_state[0]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1)
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__ThreadLocal__set_state(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__ThreadLocal__set_state", 0);

  /* "(tree fragment)":12
 *     return __pyx_result
 * cdef __pyx_unpickle__ThreadLocal__set_state(_ThreadLocal __pyx_result, __pyx_state: tuple):
 *     __pyx_result._current_cufft_callback = __pyx_state[0]             # <<<<<<<<<<<<<<
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1)
*/
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->_current_cufft_callback);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->_current_cufft_callback);
  __pyx_v___pyx_result->_current_cufft_callback = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":13
 * cdef __pyx_unpickle__ThreadLocal__set_state(_ThreadLocal __pyx_result, __pyx_state: tuple):
 *     __pyx_result._current_cufft_callback = __pyx_state[0]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1)             # <<<<<<<<<<<<<<
*/
  __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 1); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(3, 13, __pyx_L1_error)

  /* "(tree fragment)":11
 *         __pyx_unpickle__ThreadLocal__set_state(<_ThreadLocal> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__ThreadLocal__set_state(_ThreadLocal __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result._current_cufft_callback = __pyx_state[0]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__ThreadLocal__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__CallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, b'cb_load_data, cb_store_data, identity')
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_7__pyx_unpickle__CallbackManager(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_6__pyx_unpickle__CallbackManager, "__pyx_unpickle__CallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state)");
static PyMethodDef __pyx_mdef_4cupy_3fft_9_callback_7__pyx_unpickle__CallbackManager = {"__pyx_unpickle__CallbackManager", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_7__pyx_unpickle__CallbackManager, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_6__pyx_unpickle__CallbackManager};
static PyObject *__pyx_pw_4cupy_3fft_9_callback_7__pyx_unpickle__CallbackManager(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_type = 0;
  long __pyx_v___pyx_checksum;
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[3] = {0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__pyx_unpickle__CallbackManager (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 4, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  3:
        values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle__CallbackManager", 0) < (0)) __PYX_ERR(3, 4, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 3; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__CallbackManager", 1, 3, 3, i); __PYX_ERR(3, 4, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 3)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
    }
    __pyx_v___pyx_type = values[0];
    __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(3, 4, __pyx_L3_error)
    __pyx_v___pyx_state = ((PyObject*)values[2]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__CallbackManager", 1, 3, 3, __pyx_nargs); __PYX_ERR(3, 4, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__CallbackManager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(3, 4, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_6__pyx_unpickle__CallbackManager(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_6__pyx_unpickle__CallbackManager(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_v___pyx_result = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  size_t __pyx_t_4;
  int __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__CallbackManager", 0);

  /* "(tree fragment)":6
 * def __pyx_unpickle__CallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, b'cb_load_data, cb_store_data, identity')             # <<<<<<<<<<<<<<
 *     __pyx_result = _CallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:
*/
  __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, __pyx_k_cb_load_data_cb_store_data_ident); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(3, 6, __pyx_L1_error)

  /* "(tree fragment)":7
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, b'cb_load_data, cb_store_data, identity')
 *     __pyx_result = _CallbackManager.__new__(__pyx_type)             # <<<<<<<<<<<<<<
 *     if __pyx_state is not None:
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)
*/
  __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager);
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_4 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 7, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_v___pyx_result = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, b'cb_load_data, cb_store_data, identity')
 *     __pyx_result = _CallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None));
  if (__pyx_t_5) {

    /* "(tree fragment)":9
 *     __pyx_result = _CallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)             # <<<<<<<<<<<<<<
 *     return __pyx_result
 * cdef __pyx_unpickle__CallbackManager__set_state(_CallbackManager __pyx_result, __pyx_state: tuple):
*/
    if (unlikely(__pyx_v___pyx_state == Py_None)) {
      PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
      __PYX_ERR(3, 9, __pyx_L1_error)
    }
    __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__CallbackManager__set_state(((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 9, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, b'cb_load_data, cb_store_data, identity')
 *     __pyx_result = _CallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  }

  /* "(tree fragment)":10
 *     if __pyx_state is not None:
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result             # <<<<<<<<<<<<<<
 * cdef __pyx_unpickle__CallbackManager__set_state(_CallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load_data = __pyx_state[0]; __pyx_result.cb_store_data = __pyx_state[1]; __pyx_result.identity = __pyx_state[2]
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v___pyx_result);
  __pyx_r = __pyx_v___pyx_result;
  goto __pyx_L0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__CallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xfce4b78, 0xaff12da, 0x945a6fd, b'cb_load_data, cb_store_data, identity')
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__CallbackManager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v___pyx_result);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":11
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__CallbackManager__set_state(_CallbackManager __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.cb_load_data = __pyx_state[0]; __pyx_result.cb_store_data = __pyx_state[1]; __pyx_result.identity = __pyx_state[2]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 3)
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__CallbackManager__set_state(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__CallbackManager__set_state", 0);

  /* "(tree fragment)":12
 *     return __pyx_result
 * cdef __pyx_unpickle__CallbackManager__set_state(_CallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load_data = __pyx_state[0]; __pyx_result.cb_store_data = __pyx_state[1]; __pyx_result.identity = __pyx_state[2]             # <<<<<<<<<<<<<<
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 3)
*/
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->cb_load_data);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->cb_load_data);
  __pyx_v___pyx_result->cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->cb_store_data);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->cb_store_data);
  __pyx_v___pyx_result->cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->identity);
  __Pyx_DECREF(__pyx_v___pyx_result->identity);
  __pyx_v___pyx_result->identity = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":13
 * cdef __pyx_unpickle__CallbackManager__set_state(_CallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load_data = __pyx_state[0]; __pyx_result.cb_store_data = __pyx_state[1]; __pyx_result.identity = __pyx_state[2]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 3)             # <<<<<<<<<<<<<<
*/
  __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 3); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(3, 13, __pyx_L1_error)

  /* "(tree fragment)":11
 *         __pyx_unpickle__CallbackManager__set_state(<_CallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__CallbackManager__set_state(_CallbackManager __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.cb_load_data = __pyx_state[0]; __pyx_result.cb_store_data = __pyx_state[1]; __pyx_result.identity = __pyx_state[2]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 3)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__CallbackManager__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__LegacyCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_9__pyx_unpickle__LegacyCallbackManager(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_8__pyx_unpickle__LegacyCallbackManager, "__pyx_unpickle__LegacyCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state)");
static PyMethodDef __pyx_mdef_4cupy_3fft_9_callback_9__pyx_unpickle__LegacyCallbackManager = {"__pyx_unpickle__LegacyCallbackManager", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_9__pyx_unpickle__LegacyCallbackManager, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_8__pyx_unpickle__LegacyCallbackManager};
static PyObject *__pyx_pw_4cupy_3fft_9_callback_9__pyx_unpickle__LegacyCallbackManager(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_type = 0;
  long __pyx_v___pyx_checksum;
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[3] = {0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__pyx_unpickle__LegacyCallbackManager (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 4, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  3:
        values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle__LegacyCallbackManager", 0) < (0)) __PYX_ERR(3, 4, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 3; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__LegacyCallbackManager", 1, 3, 3, i); __PYX_ERR(3, 4, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 3)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
    }
    __pyx_v___pyx_type = values[0];
    __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(3, 4, __pyx_L3_error)
    __pyx_v___pyx_state = ((PyObject*)values[2]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__LegacyCallbackManager", 1, 3, 3, __pyx_nargs); __PYX_ERR(3, 4, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__LegacyCallbackManager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(3, 4, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_8__pyx_unpickle__LegacyCallbackManager(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_8__pyx_unpickle__LegacyCallbackManager(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_v___pyx_result = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  size_t __pyx_t_4;
  int __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__LegacyCallbackManager", 0);

  /* "(tree fragment)":6
 * def __pyx_unpickle__LegacyCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')             # <<<<<<<<<<<<<<
 *     __pyx_result = _LegacyCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:
*/
  __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, __pyx_k_cb_load_cb_load_data_cb_store_cb); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(3, 6, __pyx_L1_error)

  /* "(tree fragment)":7
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')
 *     __pyx_result = _LegacyCallbackManager.__new__(__pyx_type)             # <<<<<<<<<<<<<<
 *     if __pyx_state is not None:
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)
*/
  __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager);
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_4 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 7, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_v___pyx_result = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')
 *     __pyx_result = _LegacyCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None));
  if (__pyx_t_5) {

    /* "(tree fragment)":9
 *     __pyx_result = _LegacyCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)             # <<<<<<<<<<<<<<
 *     return __pyx_result
 * cdef __pyx_unpickle__LegacyCallbackManager__set_state(_LegacyCallbackManager __pyx_result, __pyx_state: tuple):
*/
    if (unlikely(__pyx_v___pyx_state == Py_None)) {
      PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
      __PYX_ERR(3, 9, __pyx_L1_error)
    }
    __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__LegacyCallbackManager__set_state(((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 9, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')
 *     __pyx_result = _LegacyCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  }

  /* "(tree fragment)":10
 *     if __pyx_state is not None:
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result             # <<<<<<<<<<<<<<
 * cdef __pyx_unpickle__LegacyCallbackManager__set_state(_LegacyCallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_store = __pyx_state[2]; __pyx_result.cb_store_data = __pyx_state[3]; __pyx_result.identity = __pyx_state[4]; __pyx_result.mod = __pyx_state[5]
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v___pyx_result);
  __pyx_r = __pyx_v___pyx_result;
  goto __pyx_L0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__LegacyCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__LegacyCallbackManager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v___pyx_result);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":11
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__LegacyCallbackManager__set_state(_LegacyCallbackManager __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_store = __pyx_state[2]; __pyx_result.cb_store_data = __pyx_state[3]; __pyx_result.identity = __pyx_state[4]; __pyx_result.mod = __pyx_state[5]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 6)
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__LegacyCallbackManager__set_state(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__LegacyCallbackManager__set_state", 0);

  /* "(tree fragment)":12
 *     return __pyx_result
 * cdef __pyx_unpickle__LegacyCallbackManager__set_state(_LegacyCallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_store = __pyx_state[2]; __pyx_result.cb_store_data = __pyx_state[3]; __pyx_result.identity = __pyx_state[4]; __pyx_result.mod = __pyx_state[5]             # <<<<<<<<<<<<<<
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 6)
*/
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_load);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_load);
  __pyx_v___pyx_result->cb_load = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_load_data);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_load_data);
  __pyx_v___pyx_result->__pyx_base.cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_store);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_store);
  __pyx_v___pyx_result->cb_store = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_store_data);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_store_data);
  __pyx_v___pyx_result->__pyx_base.cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 4, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->__pyx_base.identity);
  __Pyx_DECREF(__pyx_v___pyx_result->__pyx_base.identity);
  __pyx_v___pyx_result->__pyx_base.identity = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 5, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->mod);
  __Pyx_DECREF(__pyx_v___pyx_result->mod);
  __pyx_v___pyx_result->mod = __pyx_t_1;
  __pyx_t_1 = 0;

  /* "(tree fragment)":13
 * cdef __pyx_unpickle__LegacyCallbackManager__set_state(_LegacyCallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_store = __pyx_state[2]; __pyx_result.cb_store_data = __pyx_state[3]; __pyx_result.identity = __pyx_state[4]; __pyx_result.mod = __pyx_state[5]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 6)             # <<<<<<<<<<<<<<
*/
  __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 6); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(3, 13, __pyx_L1_error)

  /* "(tree fragment)":11
 *         __pyx_unpickle__LegacyCallbackManager__set_state(<_LegacyCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__LegacyCallbackManager__set_state(_LegacyCallbackManager __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_store = __pyx_state[2]; __pyx_result.cb_store_data = __pyx_state[3]; __pyx_result.identity = __pyx_state[4]; __pyx_result.mod = __pyx_state[5]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 6)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__LegacyCallbackManager__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__JITCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, b'cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity')
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_11__pyx_unpickle__JITCallbackManager(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_10__pyx_unpickle__JITCallbackManager, "__pyx_unpickle__JITCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state)");
static PyMethodDef __pyx_mdef_4cupy_3fft_9_callback_11__pyx_unpickle__JITCallbackManager = {"__pyx_unpickle__JITCallbackManager", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_11__pyx_unpickle__JITCallbackManager, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_10__pyx_unpickle__JITCallbackManager};
static PyObject *__pyx_pw_4cupy_3fft_9_callback_11__pyx_unpickle__JITCallbackManager(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_type = 0;
  long __pyx_v___pyx_checksum;
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[3] = {0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__pyx_unpickle__JITCallbackManager (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 4, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  3:
        values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle__JITCallbackManager", 0) < (0)) __PYX_ERR(3, 4, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 3; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__JITCallbackManager", 1, 3, 3, i); __PYX_ERR(3, 4, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 3)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
    }
    __pyx_v___pyx_type = values[0];
    __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(3, 4, __pyx_L3_error)
    __pyx_v___pyx_state = ((PyObject*)values[2]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__pyx_unpickle__JITCallbackManager", 1, 3, 3, __pyx_nargs); __PYX_ERR(3, 4, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__JITCallbackManager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(3, 4, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_10__pyx_unpickle__JITCallbackManager(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_10__pyx_unpickle__JITCallbackManager(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_v___pyx_result = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  size_t __pyx_t_4;
  int __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__JITCallbackManager", 0);

  /* "(tree fragment)":6
 * def __pyx_unpickle__JITCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, b'cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity')             # <<<<<<<<<<<<<<
 *     __pyx_result = _JITCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:
*/
  __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, __pyx_k_cb_load_cb_load_data_cb_load_lto); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(3, 6, __pyx_L1_error)

  /* "(tree fragment)":7
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, b'cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity')
 *     __pyx_result = _JITCallbackManager.__new__(__pyx_type)             # <<<<<<<<<<<<<<
 *     if __pyx_state is not None:
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)
*/
  __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager);
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_4 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 7, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_v___pyx_result = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, b'cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity')
 *     __pyx_result = _JITCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None));
  if (__pyx_t_5) {

    /* "(tree fragment)":9
 *     __pyx_result = _JITCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)             # <<<<<<<<<<<<<<
 *     return __pyx_result
 * cdef __pyx_unpickle__JITCallbackManager__set_state(_JITCallbackManager __pyx_result, __pyx_state: tuple):
*/
    if (unlikely(__pyx_v___pyx_state == Py_None)) {
      PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
      __PYX_ERR(3, 9, __pyx_L1_error)
    }
    __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle__JITCallbackManager__set_state(((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 9, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, b'cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity')
 *     __pyx_result = _JITCallbackManager.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  }

  /* "(tree fragment)":10
 *     if __pyx_state is not None:
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result             # <<<<<<<<<<<<<<
 * cdef __pyx_unpickle__JITCallbackManager__set_state(_JITCallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_load_lto = __pyx_state[2]; __pyx_result.cb_load_name = __pyx_state[3]; __pyx_result.cb_store = __pyx_state[4]; __pyx_result.cb_store_data = __pyx_state[5]; __pyx_result.cb_store_lto = __pyx_state[6]; __pyx_result.cb_store_name = __pyx_state[7]; __pyx_result.identity = __pyx_state[8]
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v___pyx_result);
  __pyx_r = __pyx_v___pyx_result;
  goto __pyx_L0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__JITCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe39deaa, 0xa8a21c4, 0xe501e24, b'cb_load, cb_load_data, cb_load_lto, cb_load_name, cb_store, cb_store_data, cb_store_lto, cb_store_name, identity')
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__JITCallbackManager", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v___pyx_result);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":11
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__JITCallbackManager__set_state(_JITCallbackManager __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_load_lto = __pyx_state[2]; __pyx_result.cb_load_name = __pyx_state[3]; __pyx_result.cb_store = __pyx_state[4]; __pyx_result.cb_store_data = __pyx_state[5]; __pyx_result.cb_store_lto = __pyx_state[6]; __pyx_result.cb_store_name = __pyx_state[7]; __pyx_result.identity = __pyx_state[8]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 9)
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle__JITCallbackManager__set_state(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle__JITCallbackManager__set_state", 0);

  /* "(tree fragment)":12
 *     return __pyx_result
 * cdef __pyx_unpickle__JITCallbackManager__set_state(_JITCallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_load_lto = __pyx_state[2]; __pyx_result.cb_load_name = __pyx_state[3]; __pyx_result.cb_store = __pyx_state[4]; __pyx_result.cb_store_data = __pyx_state[5]; __pyx_result.cb_store_lto = __pyx_state[6]; __pyx_result.cb_store_name = __pyx_state[7]; __pyx_result.identity = __pyx_state[8]             # <<<<<<<<<<<<<<
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 9)
*/
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_load);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_load);
  __pyx_v___pyx_result->cb_load = __pyx_t_1;
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_load_data);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_load_data);
  __pyx_v___pyx_result->__pyx_base.cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_load_lto);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_load_lto);
  __pyx_v___pyx_result->cb_load_lto = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_load_name);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_load_name);
  __pyx_v___pyx_result->cb_load_name = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 4, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_store);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_store);
  __pyx_v___pyx_result->cb_store = __pyx_t_1;
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 5, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_store_data);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->__pyx_base.cb_store_data);
  __pyx_v___pyx_result->__pyx_base.cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 6, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_store_lto);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_store_lto);
  __pyx_v___pyx_result->cb_store_lto = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 7, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->cb_store_name);
  __Pyx_DECREF(__pyx_v___pyx_result->cb_store_name);
  __pyx_v___pyx_result->cb_store_name = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 8, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(PyUnicode_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_t_1))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF(__pyx_v___pyx_result->__pyx_base.identity);
  __Pyx_DECREF(__pyx_v___pyx_result->__pyx_base.identity);
  __pyx_v___pyx_result->__pyx_base.identity = ((PyObject*)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":13
 * cdef __pyx_unpickle__JITCallbackManager__set_state(_JITCallbackManager __pyx_result, __pyx_state: tuple):
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_load_lto = __pyx_state[2]; __pyx_result.cb_load_name = __pyx_state[3]; __pyx_result.cb_store = __pyx_state[4]; __pyx_result.cb_store_data = __pyx_state[5]; __pyx_result.cb_store_lto = __pyx_state[6]; __pyx_result.cb_store_name = __pyx_state[7]; __pyx_result.identity = __pyx_state[8]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 9)             # <<<<<<<<<<<<<<
*/
  __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 9); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(3, 13, __pyx_L1_error)

  /* "(tree fragment)":11
 *         __pyx_unpickle__JITCallbackManager__set_state(<_JITCallbackManager> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle__JITCallbackManager__set_state(_JITCallbackManager __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.cb_load = __pyx_state[0]; __pyx_result.cb_load_data = __pyx_state[1]; __pyx_result.cb_load_lto = __pyx_state[2]; __pyx_result.cb_load_name = __pyx_state[3]; __pyx_result.cb_store = __pyx_state[4]; __pyx_result.cb_store_data = __pyx_state[5]; __pyx_result.cb_store_lto = __pyx_state[6]; __pyx_result.cb_store_name = __pyx_state[7]; __pyx_result.identity = __pyx_state[8]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 9)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle__JITCallbackManager__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle_set_cufft_callbacks(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')
*/

/* Python wrapper */
static PyObject *__pyx_pw_4cupy_3fft_9_callback_13__pyx_unpickle_set_cufft_callbacks(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
); /*proto*/
PyDoc_STRVAR(__pyx_doc_4cupy_3fft_9_callback_12__pyx_unpickle_set_cufft_callbacks, "__pyx_unpickle_set_cufft_callbacks(__pyx_type, long __pyx_checksum, tuple __pyx_state)");
static PyMethodDef __pyx_mdef_4cupy_3fft_9_callback_13__pyx_unpickle_set_cufft_callbacks = {"__pyx_unpickle_set_cufft_callbacks", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_13__pyx_unpickle_set_cufft_callbacks, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_12__pyx_unpickle_set_cufft_callbacks};
static PyObject *__pyx_pw_4cupy_3fft_9_callback_13__pyx_unpickle_set_cufft_callbacks(PyObject *__pyx_self, 
#if CYTHON_METH_FASTCALL
PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
#else
PyObject *__pyx_args, PyObject *__pyx_kwds
#endif
) {
  PyObject *__pyx_v___pyx_type = 0;
  long __pyx_v___pyx_checksum;
  PyObject *__pyx_v___pyx_state = 0;
  #if !CYTHON_METH_FASTCALL
  CYTHON_UNUSED Py_ssize_t __pyx_nargs;
  #endif
  CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
  PyObject* values[3] = {0,0,0};
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  PyObject *__pyx_r = 0;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("__pyx_unpickle_set_cufft_callbacks (wrapper)", 0);
  #if !CYTHON_METH_FASTCALL
  #if CYTHON_ASSUME_SAFE_SIZE
  __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
  #else
  __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
  #endif
  #endif
  __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
  {
    PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0};
    const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
    if (unlikely(__pyx_kwds_len < 0)) __PYX_ERR(3, 4, __pyx_L3_error)
    if (__pyx_kwds_len > 0) {
      switch (__pyx_nargs) {
        case  3:
        values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  2:
        values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  1:
        values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
        if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      const Py_ssize_t kwd_pos_args = __pyx_nargs;
      if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle_set_cufft_callbacks", 0) < (0)) __PYX_ERR(3, 4, __pyx_L3_error)
      for (Py_ssize_t i = __pyx_nargs; i < 3; i++) {
        if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_set_cufft_callbacks", 1, 3, 3, i); __PYX_ERR(3, 4, __pyx_L3_error) }
      }
    } else if (unlikely(__pyx_nargs != 3)) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(3, 4, __pyx_L3_error)
      values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
      if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(3, 4, __pyx_L3_error)
    }
    __pyx_v___pyx_type = values[0];
    __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(3, 4, __pyx_L3_error)
    __pyx_v___pyx_state = ((PyObject*)values[2]);
  }
  goto __pyx_L6_skip;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_set_cufft_callbacks", 1, 3, 3, __pyx_nargs); __PYX_ERR(3, 4, __pyx_L3_error)
  __pyx_L6_skip:;
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L3_error:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle_set_cufft_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return NULL;
  __pyx_L4_argument_unpacking_done:;
  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(3, 4, __pyx_L1_error)
  __pyx_r = __pyx_pf_4cupy_3fft_9_callback_12__pyx_unpickle_set_cufft_callbacks(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state);

  /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __pyx_r = NULL;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  goto __pyx_L7_cleaned_up;
  __pyx_L0:;
  for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
    Py_XDECREF(values[__pyx_temp]);
  }
  __pyx_L7_cleaned_up:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

static PyObject *__pyx_pf_4cupy_3fft_9_callback_12__pyx_unpickle_set_cufft_callbacks(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_v___pyx_result = 0;
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  int __pyx_t_1;
  PyObject *__pyx_t_2 = NULL;
  PyObject *__pyx_t_3 = NULL;
  size_t __pyx_t_4;
  int __pyx_t_5;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle_set_cufft_callbacks", 0);

  /* "(tree fragment)":6
 * def __pyx_unpickle_set_cufft_callbacks(__pyx_type, long __pyx_checksum, tuple __pyx_state):
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')             # <<<<<<<<<<<<<<
 *     __pyx_result = set_cufft_callbacks.__new__(__pyx_type)
 *     if __pyx_state is not None:
*/
  __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, __pyx_k_mgr_mgr_prev); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(3, 6, __pyx_L1_error)

  /* "(tree fragment)":7
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')
 *     __pyx_result = set_cufft_callbacks.__new__(__pyx_type)             # <<<<<<<<<<<<<<
 *     if __pyx_state is not None:
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)
*/
  __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks);
  __Pyx_INCREF(__pyx_t_3);
  __pyx_t_4 = 0;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type};
    __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 7, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __pyx_v___pyx_result = __pyx_t_2;
  __pyx_t_2 = 0;

  /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')
 *     __pyx_result = set_cufft_callbacks.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None));
  if (__pyx_t_5) {

    /* "(tree fragment)":9
 *     __pyx_result = set_cufft_callbacks.__new__(__pyx_type)
 *     if __pyx_state is not None:
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)             # <<<<<<<<<<<<<<
 *     return __pyx_result
 * cdef __pyx_unpickle_set_cufft_callbacks__set_state(set_cufft_callbacks __pyx_result, __pyx_state: tuple):
*/
    if (unlikely(__pyx_v___pyx_state == Py_None)) {
      PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'");
      __PYX_ERR(3, 9, __pyx_L1_error)
    }
    __pyx_t_2 = __pyx_f_4cupy_3fft_9_callback___pyx_unpickle_set_cufft_callbacks__set_state(((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 9, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

    /* "(tree fragment)":8
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')
 *     __pyx_result = set_cufft_callbacks.__new__(__pyx_type)
 *     if __pyx_state is not None:             # <<<<<<<<<<<<<<
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)
 *     return __pyx_result
*/
  }

  /* "(tree fragment)":10
 *     if __pyx_state is not None:
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)
 *     return __pyx_result             # <<<<<<<<<<<<<<
 * cdef __pyx_unpickle_set_cufft_callbacks__set_state(set_cufft_callbacks __pyx_result, __pyx_state: tuple):
 *     __pyx_result.mgr = __pyx_state[0]; __pyx_result.mgr_prev = __pyx_state[1]
*/
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_v___pyx_result);
  __pyx_r = __pyx_v___pyx_result;
  goto __pyx_L0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle_set_cufft_callbacks(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')
*/

  /* function exit code */
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_3);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle_set_cufft_callbacks", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = NULL;
  __pyx_L0:;
  __Pyx_XDECREF(__pyx_v___pyx_result);
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* "(tree fragment)":11
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle_set_cufft_callbacks__set_state(set_cufft_callbacks __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.mgr = __pyx_state[0]; __pyx_result.mgr_prev = __pyx_state[1]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 2)
*/

static PyObject *__pyx_f_4cupy_3fft_9_callback___pyx_unpickle_set_cufft_callbacks__set_state(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  PyObject *__pyx_t_1 = NULL;
  int __pyx_t_2;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__pyx_unpickle_set_cufft_callbacks__set_state", 0);

  /* "(tree fragment)":12
 *     return __pyx_result
 * cdef __pyx_unpickle_set_cufft_callbacks__set_state(set_cufft_callbacks __pyx_result, __pyx_state: tuple):
 *     __pyx_result.mgr = __pyx_state[0]; __pyx_result.mgr_prev = __pyx_state[1]             # <<<<<<<<<<<<<<
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 2)
*/
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->mgr);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->mgr);
  __pyx_v___pyx_result->mgr = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;
  __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager))))) __PYX_ERR(3, 12, __pyx_L1_error)
  __Pyx_GIVEREF(__pyx_t_1);
  __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->mgr_prev);
  __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->mgr_prev);
  __pyx_v___pyx_result->mgr_prev = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)__pyx_t_1);
  __pyx_t_1 = 0;

  /* "(tree fragment)":13
 * cdef __pyx_unpickle_set_cufft_callbacks__set_state(set_cufft_callbacks __pyx_result, __pyx_state: tuple):
 *     __pyx_result.mgr = __pyx_state[0]; __pyx_result.mgr_prev = __pyx_state[1]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 2)             # <<<<<<<<<<<<<<
*/
  __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 2); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(3, 13, __pyx_L1_error)

  /* "(tree fragment)":11
 *         __pyx_unpickle_set_cufft_callbacks__set_state(<set_cufft_callbacks> __pyx_result, __pyx_state)
 *     return __pyx_result
 * cdef __pyx_unpickle_set_cufft_callbacks__set_state(set_cufft_callbacks __pyx_result, __pyx_state: tuple):             # <<<<<<<<<<<<<<
 *     __pyx_result.mgr = __pyx_state[0]; __pyx_result.mgr_prev = __pyx_state[1]
 *     __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 2)
*/

  /* function exit code */
  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_AddTraceback("cupy.fft._callback.__pyx_unpickle_set_cufft_callbacks__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}
/* #### Code section: module_exttypes ### */
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__ThreadLocal __pyx_vtable_4cupy_3fft_9_callback__ThreadLocal;

static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__ThreadLocal(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *p;
  PyObject *o;
  o = __Pyx_AllocateExtensionType(t, 0);
  if (unlikely(!o)) return 0;
  p = ((struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)o);
  p->__pyx_vtab = __pyx_vtabptr_4cupy_3fft_9_callback__ThreadLocal;
  p->_current_cufft_callback = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None); Py_INCREF(Py_None);
  return o;
}

static void __pyx_tp_dealloc_4cupy_3fft_9_callback__ThreadLocal(PyObject *o) {
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *p = (struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)o;
  #if CYTHON_USE_TP_FINALIZE
  if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) {
    if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_4cupy_3fft_9_callback__ThreadLocal) {
      if (PyObject_CallFinalizerFromDealloc(o)) return;
    }
  }
  #endif
  PyObject_GC_UnTrack(o);
  Py_CLEAR(p->_current_cufft_callback);
  PyTypeObject *tp = Py_TYPE(o);
  #if CYTHON_USE_TYPE_SLOTS
  (*tp->tp_free)(o);
  #else
  {
    freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free);
    if (tp_free) tp_free(o);
  }
  #endif
  #if CYTHON_USE_TYPE_SPECS
  Py_DECREF(tp);
  #endif
}

static int __pyx_tp_traverse_4cupy_3fft_9_callback__ThreadLocal(PyObject *o, visitproc v, void *a) {
  int e;
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *p = (struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)o;
  {
    e = __Pyx_call_type_traverse(o, 1, v, a);
    if (e) return e;
  }
  if (p->_current_cufft_callback) {
    e = (*v)(((PyObject *)p->_current_cufft_callback), a); if (e) return e;
  }
  return 0;
}

static int __pyx_tp_clear_4cupy_3fft_9_callback__ThreadLocal(PyObject *o) {
  PyObject* tmp;
  struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *p = (struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *)o;
  tmp = ((PyObject*)p->_current_cufft_callback);
  p->_current_cufft_callback = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None); Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  return 0;
}

static PyMethodDef __pyx_methods_4cupy_3fft_9_callback__ThreadLocal[] = {
  {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_3__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_12_ThreadLocal_2__reduce_cython__},
  {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_5__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_12_ThreadLocal_4__setstate_cython__},
  {0, 0, 0, 0}
};
#if CYTHON_USE_TYPE_SPECS
static PyType_Slot __pyx_type_4cupy_3fft_9_callback__ThreadLocal_slots[] = {
  {Py_tp_dealloc, (void *)__pyx_tp_dealloc_4cupy_3fft_9_callback__ThreadLocal},
  {Py_tp_doc, (void *)PyDoc_STR("_ThreadLocal()")},
  {Py_tp_traverse, (void *)__pyx_tp_traverse_4cupy_3fft_9_callback__ThreadLocal},
  {Py_tp_clear, (void *)__pyx_tp_clear_4cupy_3fft_9_callback__ThreadLocal},
  {Py_tp_methods, (void *)__pyx_methods_4cupy_3fft_9_callback__ThreadLocal},
  {Py_tp_init, (void *)__pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_1__init__},
  {Py_tp_new, (void *)__pyx_tp_new_4cupy_3fft_9_callback__ThreadLocal},
  {0, 0},
};
static PyType_Spec __pyx_type_4cupy_3fft_9_callback__ThreadLocal_spec = {
  "cupy.fft._callback._ThreadLocal",
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal),
  0,
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC,
  __pyx_type_4cupy_3fft_9_callback__ThreadLocal_slots,
};
#else

static PyTypeObject __pyx_type_4cupy_3fft_9_callback__ThreadLocal = {
  PyVarObject_HEAD_INIT(0, 0)
  "cupy.fft._callback.""_ThreadLocal", /*tp_name*/
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal), /*tp_basicsize*/
  0, /*tp_itemsize*/
  __pyx_tp_dealloc_4cupy_3fft_9_callback__ThreadLocal, /*tp_dealloc*/
  0, /*tp_vectorcall_offset*/
  0, /*tp_getattr*/
  0, /*tp_setattr*/
  0, /*tp_as_async*/
  0, /*tp_repr*/
  0, /*tp_as_number*/
  0, /*tp_as_sequence*/
  0, /*tp_as_mapping*/
  0, /*tp_hash*/
  0, /*tp_call*/
  0, /*tp_str*/
  0, /*tp_getattro*/
  0, /*tp_setattro*/
  0, /*tp_as_buffer*/
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
  PyDoc_STR("_ThreadLocal()"), /*tp_doc*/
  __pyx_tp_traverse_4cupy_3fft_9_callback__ThreadLocal, /*tp_traverse*/
  __pyx_tp_clear_4cupy_3fft_9_callback__ThreadLocal, /*tp_clear*/
  0, /*tp_richcompare*/
  0, /*tp_weaklistoffset*/
  0, /*tp_iter*/
  0, /*tp_iternext*/
  __pyx_methods_4cupy_3fft_9_callback__ThreadLocal, /*tp_methods*/
  0, /*tp_members*/
  0, /*tp_getset*/
  0, /*tp_base*/
  0, /*tp_dict*/
  0, /*tp_descr_get*/
  0, /*tp_descr_set*/
  #if !CYTHON_USE_TYPE_SPECS
  0, /*tp_dictoffset*/
  #endif
  __pyx_pw_4cupy_3fft_9_callback_12_ThreadLocal_1__init__, /*tp_init*/
  0, /*tp_alloc*/
  __pyx_tp_new_4cupy_3fft_9_callback__ThreadLocal, /*tp_new*/
  0, /*tp_free*/
  0, /*tp_is_gc*/
  0, /*tp_bases*/
  0, /*tp_mro*/
  0, /*tp_cache*/
  0, /*tp_subclasses*/
  0, /*tp_weaklist*/
  0, /*tp_del*/
  0, /*tp_version_tag*/
  #if CYTHON_USE_TP_FINALIZE
  0, /*tp_finalize*/
  #else
  NULL, /*tp_finalize*/
  #endif
  #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800
  0, /*tp_vectorcall*/
  #endif
  #if __PYX_NEED_TP_PRINT_SLOT == 1
  0, /*tp_print*/
  #endif
  #if PY_VERSION_HEX >= 0x030C0000
  0, /*tp_watched*/
  #endif
  #if PY_VERSION_HEX >= 0x030d00A4
  0, /*tp_versions_used*/
  #endif
  #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000
  0, /*tp_pypy_flags*/
  #endif
};
#endif
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager __pyx_vtable_4cupy_3fft_9_callback__CallbackManager;

static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__CallbackManager(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) {
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *p;
  PyObject *o;
  o = __Pyx_AllocateExtensionType(t, 0);
  if (unlikely(!o)) return 0;
  p = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)o);
  p->__pyx_vtab = __pyx_vtabptr_4cupy_3fft_9_callback__CallbackManager;
  p->cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None); Py_INCREF(Py_None);
  p->cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None); Py_INCREF(Py_None);
  p->identity = ((PyObject*)Py_None); Py_INCREF(Py_None);
  return o;
}

static void __pyx_tp_dealloc_4cupy_3fft_9_callback__CallbackManager(PyObject *o) {
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)o;
  #if CYTHON_USE_TP_FINALIZE
  if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) {
    if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_4cupy_3fft_9_callback__CallbackManager) {
      if (PyObject_CallFinalizerFromDealloc(o)) return;
    }
  }
  #endif
  PyObject_GC_UnTrack(o);
  Py_CLEAR(p->cb_load_data);
  Py_CLEAR(p->cb_store_data);
  Py_CLEAR(p->identity);
  PyTypeObject *tp = Py_TYPE(o);
  #if CYTHON_USE_TYPE_SLOTS
  (*tp->tp_free)(o);
  #else
  {
    freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free);
    if (tp_free) tp_free(o);
  }
  #endif
  #if CYTHON_USE_TYPE_SPECS
  Py_DECREF(tp);
  #endif
}

static int __pyx_tp_traverse_4cupy_3fft_9_callback__CallbackManager(PyObject *o, visitproc v, void *a) {
  int e;
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)o;
  {
    e = __Pyx_call_type_traverse(o, 1, v, a);
    if (e) return e;
  }
  if (p->cb_load_data) {
    e = (*v)(((PyObject *)p->cb_load_data), a); if (e) return e;
  }
  if (p->cb_store_data) {
    e = (*v)(((PyObject *)p->cb_store_data), a); if (e) return e;
  }
  return 0;
}

static int __pyx_tp_clear_4cupy_3fft_9_callback__CallbackManager(PyObject *o) {
  PyObject* tmp;
  struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)o;
  tmp = ((PyObject*)p->cb_load_data);
  p->cb_load_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None); Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  tmp = ((PyObject*)p->cb_store_data);
  p->cb_store_data = ((struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer *)Py_None); Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  return 0;
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_16_CallbackManager_cb_load_data(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_12cb_load_data_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_16_CallbackManager_cb_store_data(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_13cb_store_data_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_16_CallbackManager_identity(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_8identity_1__get__(o);
}

static PyMethodDef __pyx_methods_4cupy_3fft_9_callback__CallbackManager[] = {
  {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_16_CallbackManager___reduce_cython__},
  {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_16_CallbackManager_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_16_CallbackManager_2__setstate_cython__},
  {0, 0, 0, 0}
};

static struct PyGetSetDef __pyx_getsets_4cupy_3fft_9_callback__CallbackManager[] = {
  {"cb_load_data", __pyx_getprop_4cupy_3fft_9_callback_16_CallbackManager_cb_load_data, 0, 0, 0},
  {"cb_store_data", __pyx_getprop_4cupy_3fft_9_callback_16_CallbackManager_cb_store_data, 0, 0, 0},
  {"identity", __pyx_getprop_4cupy_3fft_9_callback_16_CallbackManager_identity, 0, 0, 0},
  {0, 0, 0, 0, 0}
};
#if CYTHON_USE_TYPE_SPECS
static PyType_Slot __pyx_type_4cupy_3fft_9_callback__CallbackManager_slots[] = {
  {Py_tp_dealloc, (void *)__pyx_tp_dealloc_4cupy_3fft_9_callback__CallbackManager},
  {Py_tp_traverse, (void *)__pyx_tp_traverse_4cupy_3fft_9_callback__CallbackManager},
  {Py_tp_clear, (void *)__pyx_tp_clear_4cupy_3fft_9_callback__CallbackManager},
  {Py_tp_methods, (void *)__pyx_methods_4cupy_3fft_9_callback__CallbackManager},
  {Py_tp_getset, (void *)__pyx_getsets_4cupy_3fft_9_callback__CallbackManager},
  {Py_tp_new, (void *)__pyx_tp_new_4cupy_3fft_9_callback__CallbackManager},
  {0, 0},
};
static PyType_Spec __pyx_type_4cupy_3fft_9_callback__CallbackManager_spec = {
  "cupy.fft._callback._CallbackManager",
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager),
  0,
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC,
  __pyx_type_4cupy_3fft_9_callback__CallbackManager_slots,
};
#else

static PyTypeObject __pyx_type_4cupy_3fft_9_callback__CallbackManager = {
  PyVarObject_HEAD_INIT(0, 0)
  "cupy.fft._callback.""_CallbackManager", /*tp_name*/
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager), /*tp_basicsize*/
  0, /*tp_itemsize*/
  __pyx_tp_dealloc_4cupy_3fft_9_callback__CallbackManager, /*tp_dealloc*/
  0, /*tp_vectorcall_offset*/
  0, /*tp_getattr*/
  0, /*tp_setattr*/
  0, /*tp_as_async*/
  0, /*tp_repr*/
  0, /*tp_as_number*/
  0, /*tp_as_sequence*/
  0, /*tp_as_mapping*/
  0, /*tp_hash*/
  0, /*tp_call*/
  0, /*tp_str*/
  0, /*tp_getattro*/
  0, /*tp_setattro*/
  0, /*tp_as_buffer*/
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
  0, /*tp_doc*/
  __pyx_tp_traverse_4cupy_3fft_9_callback__CallbackManager, /*tp_traverse*/
  __pyx_tp_clear_4cupy_3fft_9_callback__CallbackManager, /*tp_clear*/
  0, /*tp_richcompare*/
  0, /*tp_weaklistoffset*/
  0, /*tp_iter*/
  0, /*tp_iternext*/
  __pyx_methods_4cupy_3fft_9_callback__CallbackManager, /*tp_methods*/
  0, /*tp_members*/
  __pyx_getsets_4cupy_3fft_9_callback__CallbackManager, /*tp_getset*/
  0, /*tp_base*/
  0, /*tp_dict*/
  0, /*tp_descr_get*/
  0, /*tp_descr_set*/
  #if !CYTHON_USE_TYPE_SPECS
  0, /*tp_dictoffset*/
  #endif
  0, /*tp_init*/
  0, /*tp_alloc*/
  __pyx_tp_new_4cupy_3fft_9_callback__CallbackManager, /*tp_new*/
  0, /*tp_free*/
  0, /*tp_is_gc*/
  0, /*tp_bases*/
  0, /*tp_mro*/
  0, /*tp_cache*/
  0, /*tp_subclasses*/
  0, /*tp_weaklist*/
  0, /*tp_del*/
  0, /*tp_version_tag*/
  #if CYTHON_USE_TP_FINALIZE
  0, /*tp_finalize*/
  #else
  NULL, /*tp_finalize*/
  #endif
  #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800
  0, /*tp_vectorcall*/
  #endif
  #if __PYX_NEED_TP_PRINT_SLOT == 1
  0, /*tp_print*/
  #endif
  #if PY_VERSION_HEX >= 0x030C0000
  0, /*tp_watched*/
  #endif
  #if PY_VERSION_HEX >= 0x030d00A4
  0, /*tp_versions_used*/
  #endif
  #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000
  0, /*tp_pypy_flags*/
  #endif
};
#endif
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__LegacyCallbackManager __pyx_vtable_4cupy_3fft_9_callback__LegacyCallbackManager;

static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__LegacyCallbackManager(PyTypeObject *t, PyObject *a, PyObject *k) {
  struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *p;
  PyObject *o = __pyx_tp_new_4cupy_3fft_9_callback__CallbackManager(t, a, k);
  if (unlikely(!o)) return 0;
  p = ((struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)o);
  p->__pyx_base.__pyx_vtab = (struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager*)__pyx_vtabptr_4cupy_3fft_9_callback__LegacyCallbackManager;
  p->cb_load = ((PyObject*)Py_None); Py_INCREF(Py_None);
  p->cb_store = ((PyObject*)Py_None); Py_INCREF(Py_None);
  p->mod = Py_None; Py_INCREF(Py_None);
  return o;
}

static void __pyx_tp_dealloc_4cupy_3fft_9_callback__LegacyCallbackManager(PyObject *o) {
  struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)o;
  #if CYTHON_USE_TP_FINALIZE
  if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) {
    if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_4cupy_3fft_9_callback__LegacyCallbackManager) {
      if (PyObject_CallFinalizerFromDealloc(o)) return;
    }
  }
  #endif
  PyObject_GC_UnTrack(o);
  Py_CLEAR(p->cb_load);
  Py_CLEAR(p->cb_store);
  Py_CLEAR(p->mod);
  PyObject_GC_Track(o);
  __pyx_tp_dealloc_4cupy_3fft_9_callback__CallbackManager(o);
}

static int __pyx_tp_traverse_4cupy_3fft_9_callback__LegacyCallbackManager(PyObject *o, visitproc v, void *a) {
  int e;
  struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)o;
  e = __pyx_tp_traverse_4cupy_3fft_9_callback__CallbackManager(o, v, a); if (e) return e;
  if (p->mod) {
    e = (*v)(p->mod, a); if (e) return e;
  }
  return 0;
}

static int __pyx_tp_clear_4cupy_3fft_9_callback__LegacyCallbackManager(PyObject *o) {
  PyObject* tmp;
  struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *)o;
  __pyx_tp_clear_4cupy_3fft_9_callback__CallbackManager(o);
  tmp = ((PyObject*)p->mod);
  p->mod = Py_None; Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  return 0;
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_22_LegacyCallbackManager_cb_load(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_7cb_load_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_22_LegacyCallbackManager_cb_store(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_8cb_store_1__get__(o);
}

static PyMethodDef __pyx_methods_4cupy_3fft_9_callback__LegacyCallbackManager[] = {
  {"create_plan", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_3create_plan, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_2create_plan},
  {"set_callbacks", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_5set_callbacks, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_4set_callbacks},
  {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_6__reduce_cython__},
  {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_22_LegacyCallbackManager_8__setstate_cython__},
  {0, 0, 0, 0}
};

static struct PyGetSetDef __pyx_getsets_4cupy_3fft_9_callback__LegacyCallbackManager[] = {
  {"cb_load", __pyx_getprop_4cupy_3fft_9_callback_22_LegacyCallbackManager_cb_load, 0, 0, 0},
  {"cb_store", __pyx_getprop_4cupy_3fft_9_callback_22_LegacyCallbackManager_cb_store, 0, 0, 0},
  {0, 0, 0, 0, 0}
};
#if CYTHON_USE_TYPE_SPECS
static PyType_Slot __pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager_slots[] = {
  {Py_tp_dealloc, (void *)__pyx_tp_dealloc_4cupy_3fft_9_callback__LegacyCallbackManager},
  {Py_tp_doc, (void *)PyDoc_STR("_LegacyCallbackManager(str cb_load='', str cb_store='', MemoryPointer cb_load_data=None, MemoryPointer cb_store_data=None)")},
  {Py_tp_traverse, (void *)__pyx_tp_traverse_4cupy_3fft_9_callback__LegacyCallbackManager},
  {Py_tp_clear, (void *)__pyx_tp_clear_4cupy_3fft_9_callback__LegacyCallbackManager},
  {Py_tp_methods, (void *)__pyx_methods_4cupy_3fft_9_callback__LegacyCallbackManager},
  {Py_tp_getset, (void *)__pyx_getsets_4cupy_3fft_9_callback__LegacyCallbackManager},
  {Py_tp_init, (void *)__pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_1__init__},
  {Py_tp_new, (void *)__pyx_tp_new_4cupy_3fft_9_callback__LegacyCallbackManager},
  {0, 0},
};
static PyType_Spec __pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager_spec = {
  "cupy.fft._callback._LegacyCallbackManager",
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager),
  0,
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC,
  __pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager_slots,
};
#else

static PyTypeObject __pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager = {
  PyVarObject_HEAD_INIT(0, 0)
  "cupy.fft._callback.""_LegacyCallbackManager", /*tp_name*/
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager), /*tp_basicsize*/
  0, /*tp_itemsize*/
  __pyx_tp_dealloc_4cupy_3fft_9_callback__LegacyCallbackManager, /*tp_dealloc*/
  0, /*tp_vectorcall_offset*/
  0, /*tp_getattr*/
  0, /*tp_setattr*/
  0, /*tp_as_async*/
  0, /*tp_repr*/
  0, /*tp_as_number*/
  0, /*tp_as_sequence*/
  0, /*tp_as_mapping*/
  0, /*tp_hash*/
  0, /*tp_call*/
  0, /*tp_str*/
  0, /*tp_getattro*/
  0, /*tp_setattro*/
  0, /*tp_as_buffer*/
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
  PyDoc_STR("_LegacyCallbackManager(str cb_load='', str cb_store='', MemoryPointer cb_load_data=None, MemoryPointer cb_store_data=None)"), /*tp_doc*/
  __pyx_tp_traverse_4cupy_3fft_9_callback__LegacyCallbackManager, /*tp_traverse*/
  __pyx_tp_clear_4cupy_3fft_9_callback__LegacyCallbackManager, /*tp_clear*/
  0, /*tp_richcompare*/
  0, /*tp_weaklistoffset*/
  0, /*tp_iter*/
  0, /*tp_iternext*/
  __pyx_methods_4cupy_3fft_9_callback__LegacyCallbackManager, /*tp_methods*/
  0, /*tp_members*/
  __pyx_getsets_4cupy_3fft_9_callback__LegacyCallbackManager, /*tp_getset*/
  0, /*tp_base*/
  0, /*tp_dict*/
  0, /*tp_descr_get*/
  0, /*tp_descr_set*/
  #if !CYTHON_USE_TYPE_SPECS
  0, /*tp_dictoffset*/
  #endif
  __pyx_pw_4cupy_3fft_9_callback_22_LegacyCallbackManager_1__init__, /*tp_init*/
  0, /*tp_alloc*/
  __pyx_tp_new_4cupy_3fft_9_callback__LegacyCallbackManager, /*tp_new*/
  0, /*tp_free*/
  0, /*tp_is_gc*/
  0, /*tp_bases*/
  0, /*tp_mro*/
  0, /*tp_cache*/
  0, /*tp_subclasses*/
  0, /*tp_weaklist*/
  0, /*tp_del*/
  0, /*tp_version_tag*/
  #if CYTHON_USE_TP_FINALIZE
  0, /*tp_finalize*/
  #else
  NULL, /*tp_finalize*/
  #endif
  #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800
  0, /*tp_vectorcall*/
  #endif
  #if __PYX_NEED_TP_PRINT_SLOT == 1
  0, /*tp_print*/
  #endif
  #if PY_VERSION_HEX >= 0x030C0000
  0, /*tp_watched*/
  #endif
  #if PY_VERSION_HEX >= 0x030d00A4
  0, /*tp_versions_used*/
  #endif
  #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000
  0, /*tp_pypy_flags*/
  #endif
};
#endif
static struct __pyx_vtabstruct_4cupy_3fft_9_callback__JITCallbackManager __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager;

static PyObject *__pyx_tp_new_4cupy_3fft_9_callback__JITCallbackManager(PyTypeObject *t, PyObject *a, PyObject *k) {
  struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *p;
  PyObject *o = __pyx_tp_new_4cupy_3fft_9_callback__CallbackManager(t, a, k);
  if (unlikely(!o)) return 0;
  p = ((struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)o);
  p->__pyx_base.__pyx_vtab = (struct __pyx_vtabstruct_4cupy_3fft_9_callback__CallbackManager*)__pyx_vtabptr_4cupy_3fft_9_callback__JITCallbackManager;
  p->cb_load = Py_None; Py_INCREF(Py_None);
  p->cb_store = Py_None; Py_INCREF(Py_None);
  p->cb_load_name = ((PyObject*)Py_None); Py_INCREF(Py_None);
  p->cb_store_name = ((PyObject*)Py_None); Py_INCREF(Py_None);
  p->cb_load_lto = ((PyObject*)Py_None); Py_INCREF(Py_None);
  p->cb_store_lto = ((PyObject*)Py_None); Py_INCREF(Py_None);
  return o;
}

static void __pyx_tp_dealloc_4cupy_3fft_9_callback__JITCallbackManager(PyObject *o) {
  struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)o;
  #if CYTHON_USE_TP_FINALIZE
  if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) {
    if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_4cupy_3fft_9_callback__JITCallbackManager) {
      if (PyObject_CallFinalizerFromDealloc(o)) return;
    }
  }
  #endif
  PyObject_GC_UnTrack(o);
  Py_CLEAR(p->cb_load);
  Py_CLEAR(p->cb_store);
  Py_CLEAR(p->cb_load_name);
  Py_CLEAR(p->cb_store_name);
  Py_CLEAR(p->cb_load_lto);
  Py_CLEAR(p->cb_store_lto);
  PyObject_GC_Track(o);
  __pyx_tp_dealloc_4cupy_3fft_9_callback__CallbackManager(o);
}

static int __pyx_tp_traverse_4cupy_3fft_9_callback__JITCallbackManager(PyObject *o, visitproc v, void *a) {
  int e;
  struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)o;
  e = __pyx_tp_traverse_4cupy_3fft_9_callback__CallbackManager(o, v, a); if (e) return e;
  if (p->cb_load) {
    e = (*v)(p->cb_load, a); if (e) return e;
  }
  if (p->cb_store) {
    e = (*v)(p->cb_store, a); if (e) return e;
  }
  return 0;
}

static int __pyx_tp_clear_4cupy_3fft_9_callback__JITCallbackManager(PyObject *o) {
  PyObject* tmp;
  struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *p = (struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *)o;
  __pyx_tp_clear_4cupy_3fft_9_callback__CallbackManager(o);
  tmp = ((PyObject*)p->cb_load);
  p->cb_load = Py_None; Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  tmp = ((PyObject*)p->cb_store);
  p->cb_store = Py_None; Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  return 0;
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_load(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_7cb_load_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_store(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_8cb_store_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_load_name(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_load_name_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_store_name(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_13cb_store_name_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_load_lto(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_11cb_load_lto_1__get__(o);
}

static PyObject *__pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_store_lto(PyObject *o, CYTHON_UNUSED void *x) {
  return __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_12cb_store_lto_1__get__(o);
}

static PyMethodDef __pyx_methods_4cupy_3fft_9_callback__JITCallbackManager[] = {
  {"create_plan", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_3create_plan, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_2create_plan},
  {"set_callbacks", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_5set_callbacks, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_4set_callbacks},
  {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_6__reduce_cython__},
  {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19_JITCallbackManager_8__setstate_cython__},
  {0, 0, 0, 0}
};

static struct PyGetSetDef __pyx_getsets_4cupy_3fft_9_callback__JITCallbackManager[] = {
  {"cb_load", __pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_load, 0, 0, 0},
  {"cb_store", __pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_store, 0, 0, 0},
  {"cb_load_name", __pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_load_name, 0, 0, 0},
  {"cb_store_name", __pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_store_name, 0, 0, 0},
  {"cb_load_lto", __pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_load_lto, 0, 0, 0},
  {"cb_store_lto", __pyx_getprop_4cupy_3fft_9_callback_19_JITCallbackManager_cb_store_lto, 0, 0, 0},
  {0, 0, 0, 0, 0}
};
#if CYTHON_USE_TYPE_SPECS
static PyType_Slot __pyx_type_4cupy_3fft_9_callback__JITCallbackManager_slots[] = {
  {Py_tp_dealloc, (void *)__pyx_tp_dealloc_4cupy_3fft_9_callback__JITCallbackManager},
  {Py_tp_doc, (void *)PyDoc_STR("_JITCallbackManager(cb_load=None, cb_store=None, str cb_load_name=None, str cb_store_name=None, MemoryPointer cb_load_data=None, MemoryPointer cb_store_data=None, tuple nvrtc_options=())")},
  {Py_tp_traverse, (void *)__pyx_tp_traverse_4cupy_3fft_9_callback__JITCallbackManager},
  {Py_tp_clear, (void *)__pyx_tp_clear_4cupy_3fft_9_callback__JITCallbackManager},
  {Py_tp_methods, (void *)__pyx_methods_4cupy_3fft_9_callback__JITCallbackManager},
  {Py_tp_getset, (void *)__pyx_getsets_4cupy_3fft_9_callback__JITCallbackManager},
  {Py_tp_init, (void *)__pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_1__init__},
  {Py_tp_new, (void *)__pyx_tp_new_4cupy_3fft_9_callback__JITCallbackManager},
  {0, 0},
};
static PyType_Spec __pyx_type_4cupy_3fft_9_callback__JITCallbackManager_spec = {
  "cupy.fft._callback._JITCallbackManager",
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager),
  0,
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC,
  __pyx_type_4cupy_3fft_9_callback__JITCallbackManager_slots,
};
#else

static PyTypeObject __pyx_type_4cupy_3fft_9_callback__JITCallbackManager = {
  PyVarObject_HEAD_INIT(0, 0)
  "cupy.fft._callback.""_JITCallbackManager", /*tp_name*/
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager), /*tp_basicsize*/
  0, /*tp_itemsize*/
  __pyx_tp_dealloc_4cupy_3fft_9_callback__JITCallbackManager, /*tp_dealloc*/
  0, /*tp_vectorcall_offset*/
  0, /*tp_getattr*/
  0, /*tp_setattr*/
  0, /*tp_as_async*/
  0, /*tp_repr*/
  0, /*tp_as_number*/
  0, /*tp_as_sequence*/
  0, /*tp_as_mapping*/
  0, /*tp_hash*/
  0, /*tp_call*/
  0, /*tp_str*/
  0, /*tp_getattro*/
  0, /*tp_setattro*/
  0, /*tp_as_buffer*/
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
  PyDoc_STR("_JITCallbackManager(cb_load=None, cb_store=None, str cb_load_name=None, str cb_store_name=None, MemoryPointer cb_load_data=None, MemoryPointer cb_store_data=None, tuple nvrtc_options=())"), /*tp_doc*/
  __pyx_tp_traverse_4cupy_3fft_9_callback__JITCallbackManager, /*tp_traverse*/
  __pyx_tp_clear_4cupy_3fft_9_callback__JITCallbackManager, /*tp_clear*/
  0, /*tp_richcompare*/
  0, /*tp_weaklistoffset*/
  0, /*tp_iter*/
  0, /*tp_iternext*/
  __pyx_methods_4cupy_3fft_9_callback__JITCallbackManager, /*tp_methods*/
  0, /*tp_members*/
  __pyx_getsets_4cupy_3fft_9_callback__JITCallbackManager, /*tp_getset*/
  0, /*tp_base*/
  0, /*tp_dict*/
  0, /*tp_descr_get*/
  0, /*tp_descr_set*/
  #if !CYTHON_USE_TYPE_SPECS
  0, /*tp_dictoffset*/
  #endif
  __pyx_pw_4cupy_3fft_9_callback_19_JITCallbackManager_1__init__, /*tp_init*/
  0, /*tp_alloc*/
  __pyx_tp_new_4cupy_3fft_9_callback__JITCallbackManager, /*tp_new*/
  0, /*tp_free*/
  0, /*tp_is_gc*/
  0, /*tp_bases*/
  0, /*tp_mro*/
  0, /*tp_cache*/
  0, /*tp_subclasses*/
  0, /*tp_weaklist*/
  0, /*tp_del*/
  0, /*tp_version_tag*/
  #if CYTHON_USE_TP_FINALIZE
  0, /*tp_finalize*/
  #else
  NULL, /*tp_finalize*/
  #endif
  #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800
  0, /*tp_vectorcall*/
  #endif
  #if __PYX_NEED_TP_PRINT_SLOT == 1
  0, /*tp_print*/
  #endif
  #if PY_VERSION_HEX >= 0x030C0000
  0, /*tp_watched*/
  #endif
  #if PY_VERSION_HEX >= 0x030d00A4
  0, /*tp_versions_used*/
  #endif
  #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000
  0, /*tp_pypy_flags*/
  #endif
};
#endif

static PyObject *__pyx_tp_new_4cupy_3fft_9_callback_set_cufft_callbacks(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) {
  struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *p;
  PyObject *o;
  o = __Pyx_AllocateExtensionType(t, 0);
  if (unlikely(!o)) return 0;
  p = ((struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)o);
  p->mgr = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None); Py_INCREF(Py_None);
  p->mgr_prev = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None); Py_INCREF(Py_None);
  return o;
}

static void __pyx_tp_dealloc_4cupy_3fft_9_callback_set_cufft_callbacks(PyObject *o) {
  struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *p = (struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)o;
  #if CYTHON_USE_TP_FINALIZE
  if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) {
    if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_4cupy_3fft_9_callback_set_cufft_callbacks) {
      if (PyObject_CallFinalizerFromDealloc(o)) return;
    }
  }
  #endif
  PyObject_GC_UnTrack(o);
  Py_CLEAR(p->mgr);
  Py_CLEAR(p->mgr_prev);
  PyTypeObject *tp = Py_TYPE(o);
  #if CYTHON_USE_TYPE_SLOTS
  (*tp->tp_free)(o);
  #else
  {
    freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free);
    if (tp_free) tp_free(o);
  }
  #endif
  #if CYTHON_USE_TYPE_SPECS
  Py_DECREF(tp);
  #endif
}

static int __pyx_tp_traverse_4cupy_3fft_9_callback_set_cufft_callbacks(PyObject *o, visitproc v, void *a) {
  int e;
  struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *p = (struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)o;
  {
    e = __Pyx_call_type_traverse(o, 1, v, a);
    if (e) return e;
  }
  if (p->mgr) {
    e = (*v)(((PyObject *)p->mgr), a); if (e) return e;
  }
  if (p->mgr_prev) {
    e = (*v)(((PyObject *)p->mgr_prev), a); if (e) return e;
  }
  return 0;
}

static int __pyx_tp_clear_4cupy_3fft_9_callback_set_cufft_callbacks(PyObject *o) {
  PyObject* tmp;
  struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *p = (struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks *)o;
  tmp = ((PyObject*)p->mgr);
  p->mgr = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None); Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  tmp = ((PyObject*)p->mgr_prev);
  p->mgr_prev = ((struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *)Py_None); Py_INCREF(Py_None);
  Py_XDECREF(tmp);
  return 0;
}

static PyMethodDef __pyx_methods_4cupy_3fft_9_callback_set_cufft_callbacks[] = {
  {"__enter__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_3__enter__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_2__enter__},
  {"__exit__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_5__exit__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_4__exit__},
  {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_6__reduce_cython__},
  {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cupy_3fft_9_callback_19set_cufft_callbacks_8__setstate_cython__},
  {0, 0, 0, 0}
};
#if CYTHON_USE_TYPE_SPECS
static PyType_Slot __pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks_slots[] = {
  {Py_tp_dealloc, (void *)__pyx_tp_dealloc_4cupy_3fft_9_callback_set_cufft_callbacks},
  {Py_tp_doc, (void *)PyDoc_STR("set_cufft_callbacks(cb_load=None, cb_store=None, _ndarray_base cb_load_aux_arr=None, *, _ndarray_base cb_store_aux_arr=None, str cb_load_name=None, str cb_store_name=None, MemoryPointer cb_load_data=None, MemoryPointer cb_store_data=None, str cb_ver='legacy', tuple nvrtc_options=())\n\nA context manager for setting up load and/or store callbacks.\nAny FFT calls living in this context will have callbacks set up.\n\nArgs:\n    cb_load (str): A string contains the device kernel for the load\n        callback. It must define ``d_loadCallbackPtr``.\n    cb_store (str): A string contains the device kernel for the store\n        callback. It must define ``d_storeCallbackPtr``.\n    cb_load_aux_arr (cupy.ndarray, optional): A CuPy array containing\n        data to be used in the load callback. **DEPRECATED.**\n    cb_store_aux_arr (cupy.ndarray, optional): A CuPy array containing\n        data to be used in the store callback. **DEPRECATED.**\n    cb_load_name (str): A string contains device kernel for the load\n        callback. If not defined, we attempt to infer from the provided\n        ``cb_load`` but it may fail. Only needed when using\n        ``cb_ver=\"jit\"``.\n    cb_store_name (str): A string contains device kernel for the store\n        callback. If not defined, we attempt to infer from the provided\n        ``cb_store`` but it may fail. Only needed when using\n        ``cb_ver=\"jit\"``.\n    cb_load_data (MemoryPointer, optional): A memory chunk containing\n        data to be used in the load callback.\n    cb_store_data (MemoryPointer, optional): A memory chunk containing\n        data to be used in the store callback.\n    cb_ver (str): Which cuFFT callback support to use. The default is\n        ``\"legacy\"``. Starting CUDA 12.2, ``\"jit\"`` is supported.\n\n.. note::\n    Callbacks only work for transforms over contiguous axes; the behavior\n    for non-contiguous transforms is in general undefined.\n\nBelow is the documentation only applicable to the *""jit* option.\n\nTODO\n\nBelow is the documentation only applicable to the *legacy* option.\n\n.. note::\n    An example for a load callback is shown below:\n\n    .. code-block:: python\n\n        code = r'''\n        __device__ cufftComplex CB_ConvertInputC(\n            void *dataIn,\n            size_t offset,\n            void *callerInfo,\n            void *sharedPtr) {\n          // implementation\n        }\n\n        __device__ cufftCallbackLoadC d_loadCallbackPtr = CB_ConvertInputC;\n        '''\n\n        with cp.fft.config.set_cufft_callbacks(cb_load=code):\n            out_arr = cp.fft.fft(in_arr, ...)\n\n.. note::\n    Below are the *runtime* requirements for using this feature:\n\n        * cython >= 0.29.0\n        * A host compiler that supports C++11 and above; might need to set\n          up the ``CXX`` environment variable.\n        * ``nvcc`` and the full CUDA Toolkit. Note that the ``cudatoolkit``\n          package from Conda-Forge is not enough, as it does not contain\n          static libraries.\n\n.. warning::\n    Using cuFFT callbacks requires compiling and loading a Python module at\n    runtime as well as static linking for each distinct transform and\n    callback, so the first invocation for each combination will be very\n    slow. This is a limitation of cuFFT, so use this feature only when the\n    callback-enabled transform is known more performant and can be reused\n    to amortize the cost.\n\n.. warning::\n    The generated Python modules are by default cached in\n    ``~/.cupy/callback_cache`` for possible reuse (with the same set of\n    load/store callbacks). Due to static linking, however, the file sizes\n    can be excessive! The cache position can be changed via setting\n    ``CUPY_CACHE_DIR``.\n\n.. seealso:: `cuFFT Callback Routines`_\n\n.. _cuFFT Callback Routines:\n    https://docs.nvidia.com/cuda/cufft/index.html#cufft-callback-routines")},
  {Py_tp_traverse, (void *)__pyx_tp_traverse_4cupy_3fft_9_callback_set_cufft_callbacks},
  {Py_tp_clear, (void *)__pyx_tp_clear_4cupy_3fft_9_callback_set_cufft_callbacks},
  {Py_tp_methods, (void *)__pyx_methods_4cupy_3fft_9_callback_set_cufft_callbacks},
  {Py_tp_init, (void *)__pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_1__init__},
  {Py_tp_new, (void *)__pyx_tp_new_4cupy_3fft_9_callback_set_cufft_callbacks},
  {0, 0},
};
static PyType_Spec __pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks_spec = {
  "cupy.fft._callback.set_cufft_callbacks",
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks),
  0,
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC,
  __pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks_slots,
};
#else

static PyTypeObject __pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks = {
  PyVarObject_HEAD_INIT(0, 0)
  "cupy.fft._callback.""set_cufft_callbacks", /*tp_name*/
  sizeof(struct __pyx_obj_4cupy_3fft_9_callback_set_cufft_callbacks), /*tp_basicsize*/
  0, /*tp_itemsize*/
  __pyx_tp_dealloc_4cupy_3fft_9_callback_set_cufft_callbacks, /*tp_dealloc*/
  0, /*tp_vectorcall_offset*/
  0, /*tp_getattr*/
  0, /*tp_setattr*/
  0, /*tp_as_async*/
  0, /*tp_repr*/
  0, /*tp_as_number*/
  0, /*tp_as_sequence*/
  0, /*tp_as_mapping*/
  0, /*tp_hash*/
  0, /*tp_call*/
  0, /*tp_str*/
  0, /*tp_getattro*/
  0, /*tp_setattro*/
  0, /*tp_as_buffer*/
  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
  PyDoc_STR("set_cufft_callbacks(cb_load=None, cb_store=None, _ndarray_base cb_load_aux_arr=None, *, _ndarray_base cb_store_aux_arr=None, str cb_load_name=None, str cb_store_name=None, MemoryPointer cb_load_data=None, MemoryPointer cb_store_data=None, str cb_ver='legacy', tuple nvrtc_options=())\n\nA context manager for setting up load and/or store callbacks.\nAny FFT calls living in this context will have callbacks set up.\n\nArgs:\n    cb_load (str): A string contains the device kernel for the load\n        callback. It must define ``d_loadCallbackPtr``.\n    cb_store (str): A string contains the device kernel for the store\n        callback. It must define ``d_storeCallbackPtr``.\n    cb_load_aux_arr (cupy.ndarray, optional): A CuPy array containing\n        data to be used in the load callback. **DEPRECATED.**\n    cb_store_aux_arr (cupy.ndarray, optional): A CuPy array containing\n        data to be used in the store callback. **DEPRECATED.**\n    cb_load_name (str): A string contains device kernel for the load\n        callback. If not defined, we attempt to infer from the provided\n        ``cb_load`` but it may fail. Only needed when using\n        ``cb_ver=\"jit\"``.\n    cb_store_name (str): A string contains device kernel for the store\n        callback. If not defined, we attempt to infer from the provided\n        ``cb_store`` but it may fail. Only needed when using\n        ``cb_ver=\"jit\"``.\n    cb_load_data (MemoryPointer, optional): A memory chunk containing\n        data to be used in the load callback.\n    cb_store_data (MemoryPointer, optional): A memory chunk containing\n        data to be used in the store callback.\n    cb_ver (str): Which cuFFT callback support to use. The default is\n        ``\"legacy\"``. Starting CUDA 12.2, ``\"jit\"`` is supported.\n\n.. note::\n    Callbacks only work for transforms over contiguous axes; the behavior\n    for non-contiguous transforms is in general undefined.\n\nBelow is the documentation only applicable to the *""jit* option.\n\nTODO\n\nBelow is the documentation only applicable to the *legacy* option.\n\n.. note::\n    An example for a load callback is shown below:\n\n    .. code-block:: python\n\n        code = r'''\n        __device__ cufftComplex CB_ConvertInputC(\n            void *dataIn,\n            size_t offset,\n            void *callerInfo,\n            void *sharedPtr) {\n          // implementation\n        }\n\n        __device__ cufftCallbackLoadC d_loadCallbackPtr = CB_ConvertInputC;\n        '''\n\n        with cp.fft.config.set_cufft_callbacks(cb_load=code):\n            out_arr = cp.fft.fft(in_arr, ...)\n\n.. note::\n    Below are the *runtime* requirements for using this feature:\n\n        * cython >= 0.29.0\n        * A host compiler that supports C++11 and above; might need to set\n          up the ``CXX`` environment variable.\n        * ``nvcc`` and the full CUDA Toolkit. Note that the ``cudatoolkit``\n          package from Conda-Forge is not enough, as it does not contain\n          static libraries.\n\n.. warning::\n    Using cuFFT callbacks requires compiling and loading a Python module at\n    runtime as well as static linking for each distinct transform and\n    callback, so the first invocation for each combination will be very\n    slow. This is a limitation of cuFFT, so use this feature only when the\n    callback-enabled transform is known more performant and can be reused\n    to amortize the cost.\n\n.. warning::\n    The generated Python modules are by default cached in\n    ``~/.cupy/callback_cache`` for possible reuse (with the same set of\n    load/store callbacks). Due to static linking, however, the file sizes\n    can be excessive! The cache position can be changed via setting\n    ``CUPY_CACHE_DIR``.\n\n.. seealso:: `cuFFT Callback Routines`_\n\n.. _cuFFT Callback Routines:\n    https://docs.nvidia.com/cuda/cufft/index.html#cufft-callback-routines"), /*tp_doc*/
  __pyx_tp_traverse_4cupy_3fft_9_callback_set_cufft_callbacks, /*tp_traverse*/
  __pyx_tp_clear_4cupy_3fft_9_callback_set_cufft_callbacks, /*tp_clear*/
  0, /*tp_richcompare*/
  0, /*tp_weaklistoffset*/
  0, /*tp_iter*/
  0, /*tp_iternext*/
  __pyx_methods_4cupy_3fft_9_callback_set_cufft_callbacks, /*tp_methods*/
  0, /*tp_members*/
  0, /*tp_getset*/
  0, /*tp_base*/
  0, /*tp_dict*/
  0, /*tp_descr_get*/
  0, /*tp_descr_set*/
  #if !CYTHON_USE_TYPE_SPECS
  0, /*tp_dictoffset*/
  #endif
  __pyx_pw_4cupy_3fft_9_callback_19set_cufft_callbacks_1__init__, /*tp_init*/
  0, /*tp_alloc*/
  __pyx_tp_new_4cupy_3fft_9_callback_set_cufft_callbacks, /*tp_new*/
  0, /*tp_free*/
  0, /*tp_is_gc*/
  0, /*tp_bases*/
  0, /*tp_mro*/
  0, /*tp_cache*/
  0, /*tp_subclasses*/
  0, /*tp_weaklist*/
  0, /*tp_del*/
  0, /*tp_version_tag*/
  #if CYTHON_USE_TP_FINALIZE
  0, /*tp_finalize*/
  #else
  NULL, /*tp_finalize*/
  #endif
  #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800
  0, /*tp_vectorcall*/
  #endif
  #if __PYX_NEED_TP_PRINT_SLOT == 1
  0, /*tp_print*/
  #endif
  #if PY_VERSION_HEX >= 0x030C0000
  0, /*tp_watched*/
  #endif
  #if PY_VERSION_HEX >= 0x030d00A4
  0, /*tp_versions_used*/
  #endif
  #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000
  0, /*tp_pypy_flags*/
  #endif
};
#endif

static PyMethodDef __pyx_methods[] = {
  {"get_current_callback_manager", (PyCFunction)__pyx_pw_4cupy_3fft_9_callback_3get_current_callback_manager, METH_NOARGS, __pyx_doc_4cupy_3fft_9_callback_2get_current_callback_manager},
  {0, 0, 0, 0}
};
/* #### Code section: initfunc_declarations ### */
static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/
/* #### Code section: init_module ### */

static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0);
  /*--- Global init code ---*/
  __pyx_v_4cupy_3fft_9_callback__cc = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__python_include = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__nvcc = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__cuda_path = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__cuda_include = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__nvprune = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__build_ver = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__cupy_root = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__cupy_include = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__source_dir = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__ext_suffix = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__callback_dev_code = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__callback_cache_dir = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__callback_mgr = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__callback_thread_local = Py_None; Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback__cc_major_map = ((PyObject*)Py_None); Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback_cupy_load_callback_pattern = Py_None; Py_INCREF(Py_None);
  __pyx_v_4cupy_3fft_9_callback_cupy_store_callback_pattern = Py_None; Py_INCREF(Py_None);
  __Pyx_RefNannyFinishContext();
  return 0;
}

static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0);
  /*--- Variable export code ---*/
  __Pyx_RefNannyFinishContext();
  return 0;
}

static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0);
  /*--- Function export code ---*/
  __Pyx_RefNannyFinishContext();
  return 0;
}

static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0);
  /*--- Type init code ---*/
  __pyx_vtabptr_4cupy_3fft_9_callback__ThreadLocal = &__pyx_vtable_4cupy_3fft_9_callback__ThreadLocal;
  __pyx_vtable_4cupy_3fft_9_callback__ThreadLocal.get = (struct __pyx_obj_4cupy_3fft_9_callback__ThreadLocal *(*)(void))__pyx_f_4cupy_3fft_9_callback_12_ThreadLocal_get;
  #if CYTHON_USE_TYPE_SPECS
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_4cupy_3fft_9_callback__ThreadLocal_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal)) __PYX_ERR(0, 53, __pyx_L1_error)
  if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_4cupy_3fft_9_callback__ThreadLocal_spec, __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal) < (0)) __PYX_ERR(0, 53, __pyx_L1_error)
  #else
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal = &__pyx_type_4cupy_3fft_9_callback__ThreadLocal;
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  #endif
  #if !CYTHON_USE_TYPE_SPECS
  if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal) < (0)) __PYX_ERR(0, 53, __pyx_L1_error)
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000
  PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal);
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal->tp_dictoffset && __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal->tp_getattro == PyObject_GenericGetAttr)) {
    __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal->tp_getattro = PyObject_GenericGetAttr;
  }
  #endif
  if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal, __pyx_vtabptr_4cupy_3fft_9_callback__ThreadLocal) < (0)) __PYX_ERR(0, 53, __pyx_L1_error)
  if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal) < (0)) __PYX_ERR(0, 53, __pyx_L1_error)
  if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_ThreadLocal, (PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal) < (0)) __PYX_ERR(0, 53, __pyx_L1_error)
  if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__ThreadLocal) < (0)) __PYX_ERR(0, 53, __pyx_L1_error)
  __pyx_vtabptr_4cupy_3fft_9_callback__CallbackManager = &__pyx_vtable_4cupy_3fft_9_callback__CallbackManager;
  __pyx_vtable_4cupy_3fft_9_callback__CallbackManager.set_caller_infos = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__CallbackManager *, struct __pyx_opt_args_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos *__pyx_optional_args))__pyx_f_4cupy_3fft_9_callback_16_CallbackManager_set_caller_infos;
  #if CYTHON_USE_TYPE_SPECS
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_4cupy_3fft_9_callback__CallbackManager_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager)) __PYX_ERR(0, 341, __pyx_L1_error)
  if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_4cupy_3fft_9_callback__CallbackManager_spec, __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager) < (0)) __PYX_ERR(0, 341, __pyx_L1_error)
  #else
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager = &__pyx_type_4cupy_3fft_9_callback__CallbackManager;
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  #endif
  #if !CYTHON_USE_TYPE_SPECS
  if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager) < (0)) __PYX_ERR(0, 341, __pyx_L1_error)
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000
  PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager);
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager->tp_dictoffset && __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager->tp_getattro == PyObject_GenericGetAttr)) {
    __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager->tp_getattro = PyObject_GenericGetAttr;
  }
  #endif
  if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager, __pyx_vtabptr_4cupy_3fft_9_callback__CallbackManager) < (0)) __PYX_ERR(0, 341, __pyx_L1_error)
  if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager) < (0)) __PYX_ERR(0, 341, __pyx_L1_error)
  if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_CallbackManager, (PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager) < (0)) __PYX_ERR(0, 341, __pyx_L1_error)
  if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager) < (0)) __PYX_ERR(0, 341, __pyx_L1_error)
  __pyx_vtabptr_4cupy_3fft_9_callback__LegacyCallbackManager = &__pyx_vtable_4cupy_3fft_9_callback__LegacyCallbackManager;
  __pyx_vtable_4cupy_3fft_9_callback__LegacyCallbackManager.__pyx_base = *__pyx_vtabptr_4cupy_3fft_9_callback__CallbackManager;
  __pyx_vtable_4cupy_3fft_9_callback__LegacyCallbackManager.create_plan = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *, PyObject *, int __pyx_skip_dispatch))__pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_create_plan;
  __pyx_vtable_4cupy_3fft_9_callback__LegacyCallbackManager.set_callbacks = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__LegacyCallbackManager *, PyObject *, int __pyx_skip_dispatch))__pyx_f_4cupy_3fft_9_callback_22_LegacyCallbackManager_set_callbacks;
  #if CYTHON_USE_TYPE_SPECS
  __pyx_t_1 = PyTuple_Pack(1, (PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 369, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager_spec, __pyx_t_1);
  __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (unlikely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager)) __PYX_ERR(0, 369, __pyx_L1_error)
  if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager_spec, __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager) < (0)) __PYX_ERR(0, 369, __pyx_L1_error)
  #else
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager = &__pyx_type_4cupy_3fft_9_callback__LegacyCallbackManager;
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager->tp_base = __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager;
  #endif
  #if !CYTHON_USE_TYPE_SPECS
  if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager) < (0)) __PYX_ERR(0, 369, __pyx_L1_error)
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000
  PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager);
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager->tp_dictoffset && __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager->tp_getattro == PyObject_GenericGetAttr)) {
    __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager->tp_getattro = PyObject_GenericGetAttr;
  }
  #endif
  if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager, __pyx_vtabptr_4cupy_3fft_9_callback__LegacyCallbackManager) < (0)) __PYX_ERR(0, 369, __pyx_L1_error)
  if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager) < (0)) __PYX_ERR(0, 369, __pyx_L1_error)
  if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_LegacyCallbackManager, (PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager) < (0)) __PYX_ERR(0, 369, __pyx_L1_error)
  if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__LegacyCallbackManager) < (0)) __PYX_ERR(0, 369, __pyx_L1_error)
  __pyx_vtabptr_4cupy_3fft_9_callback__JITCallbackManager = &__pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager;
  __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager.__pyx_base = *__pyx_vtabptr_4cupy_3fft_9_callback__CallbackManager;
  __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager._get_cuda_include = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *))__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager__get_cuda_include;
  __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager._sanity_checks = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *, PyObject *, PyObject *, PyObject *))__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager__sanity_checks;
  __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager.compile_lto = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *, PyObject *))__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_compile_lto;
  __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager.create_plan = (PyObject *(*)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, intptr_t, PyObject *, int __pyx_skip_dispatch))__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_create_plan;
  __pyx_vtable_4cupy_3fft_9_callback__JITCallbackManager.set_callbacks = (intptr_t (*)(struct __pyx_obj_4cupy_3fft_9_callback__JITCallbackManager *, PyObject *, int __pyx_skip_dispatch))__pyx_f_4cupy_3fft_9_callback_19_JITCallbackManager_set_callbacks;
  #if CYTHON_USE_TYPE_SPECS
  __pyx_t_1 = PyTuple_Pack(1, (PyObject *)__pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 536, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_4cupy_3fft_9_callback__JITCallbackManager_spec, __pyx_t_1);
  __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
  if (unlikely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager)) __PYX_ERR(0, 536, __pyx_L1_error)
  if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_4cupy_3fft_9_callback__JITCallbackManager_spec, __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager) < (0)) __PYX_ERR(0, 536, __pyx_L1_error)
  #else
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager = &__pyx_type_4cupy_3fft_9_callback__JITCallbackManager;
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager->tp_base = __pyx_mstate_global->__pyx_ptype_4cupy_3fft_9_callback__CallbackManager;
  #endif
  #if !CYTHON_USE_TYPE_SPECS
  if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager) < (0)) __PYX_ERR(0, 536, __pyx_L1_error)
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000
  PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager);
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager->tp_dictoffset && __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager->tp_getattro == PyObject_GenericGetAttr)) {
    __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager->tp_getattro = PyObject_GenericGetAttr;
  }
  #endif
  if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager, __pyx_vtabptr_4cupy_3fft_9_callback__JITCallbackManager) < (0)) __PYX_ERR(0, 536, __pyx_L1_error)
  if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager) < (0)) __PYX_ERR(0, 536, __pyx_L1_error)
  if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_JITCallbackManager, (PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager) < (0)) __PYX_ERR(0, 536, __pyx_L1_error)
  if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback__JITCallbackManager) < (0)) __PYX_ERR(0, 536, __pyx_L1_error)
  #if CYTHON_USE_TYPE_SPECS
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks)) __PYX_ERR(0, 698, __pyx_L1_error)
  if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks_spec, __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks) < (0)) __PYX_ERR(0, 698, __pyx_L1_error)
  #else
  __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks = &__pyx_type_4cupy_3fft_9_callback_set_cufft_callbacks;
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  #endif
  #if !CYTHON_USE_TYPE_SPECS
  if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks) < (0)) __PYX_ERR(0, 698, __pyx_L1_error)
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000
  PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks);
  #endif
  #if !CYTHON_COMPILING_IN_LIMITED_API
  if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks->tp_dictoffset && __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks->tp_getattro == PyObject_GenericGetAttr)) {
    __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks->tp_getattro = PyObject_GenericGetAttr;
  }
  #endif
  if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_set_cufft_callbacks, (PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks) < (0)) __PYX_ERR(0, 698, __pyx_L1_error)
  if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_4cupy_3fft_9_callback_set_cufft_callbacks) < (0)) __PYX_ERR(0, 698, __pyx_L1_error)
  __Pyx_RefNannyFinishContext();
  return 0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_RefNannyFinishContext();
  return -1;
}

static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0);
  /*--- Type import code ---*/
  __pyx_t_1 = PyImport_ImportModule("cupy_backends.cuda.api.runtime"); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 8, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_PointerAttributes = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy_backends.cuda.api.runtime", "PointerAttributes",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes),
  #else
  sizeof(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_PointerAttributes),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_PointerAttributes) __PYX_ERR(4, 8, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_MemPoolProps = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy_backends.cuda.api.runtime", "MemPoolProps",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps),
  #else
  sizeof(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_13cupy_backends_4cuda_3api_7runtime_MemPoolProps),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_13cupy_backends_4cuda_3api_7runtime_MemPoolProps) __PYX_ERR(4, 15, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyImport_ImportModule("cupy.cuda.device"); if (unlikely(!__pyx_t_1)) __PYX_ERR(5, 13, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6device_Handle = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.device", "Handle",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6device_Handle), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6device_Handle),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6device_Handle), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6device_Handle),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6device_Handle), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6device_Handle),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6device_Handle) __PYX_ERR(5, 13, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6device_Device = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.device", "Device",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6device_Device), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6device_Device),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6device_Device), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6device_Device),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6device_Device), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6device_Device),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6device_Device) __PYX_ERR(5, 18, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_6device_Device = (struct __pyx_vtabstruct_4cupy_4cuda_6device_Device*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_6device_Device); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_6device_Device)) __PYX_ERR(5, 18, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyImport_ImportModule("cupy.cuda.memory"); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 10, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_BaseMemory = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "BaseMemory",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_BaseMemory),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_BaseMemory) __PYX_ERR(1, 10, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_SystemMemory = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "SystemMemory",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_SystemMemory),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_SystemMemory) __PYX_ERR(1, 19, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_6memory_SystemMemory = (struct __pyx_vtabstruct_4cupy_4cuda_6memory_SystemMemory*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_SystemMemory); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_6memory_SystemMemory)) __PYX_ERR(1, 19, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "MemoryPointer",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPointer),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer) __PYX_ERR(1, 29, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_6memory_MemoryPointer = (struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPointer*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_MemoryPointer); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_6memory_MemoryPointer)) __PYX_ERR(1, 29, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_MemoryPool = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "MemoryPool",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_MemoryPool),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_MemoryPool) __PYX_ERR(1, 58, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_6memory_MemoryPool = (struct __pyx_vtabstruct_4cupy_4cuda_6memory_MemoryPool*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_MemoryPool); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_6memory_MemoryPool)) __PYX_ERR(1, 58, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocatorMemory = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "CFunctionAllocatorMemory",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocatorMemory),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocatorMemory) __PYX_ERR(1, 89, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocator = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "CFunctionAllocator",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_CFunctionAllocator),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocator) __PYX_ERR(1, 97, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_6memory_CFunctionAllocator = (struct __pyx_vtabstruct_4cupy_4cuda_6memory_CFunctionAllocator*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_CFunctionAllocator); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_6memory_CFunctionAllocator)) __PYX_ERR(1, 97, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "PythonFunctionAllocatorMemory",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocatorMemory) __PYX_ERR(1, 108, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocator = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.memory", "PythonFunctionAllocator",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_6memory_PythonFunctionAllocator),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocator) __PYX_ERR(1, 114, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_6memory_PythonFunctionAllocator = (struct __pyx_vtabstruct_4cupy_4cuda_6memory_PythonFunctionAllocator*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_6memory_PythonFunctionAllocator); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_6memory_PythonFunctionAllocator)) __PYX_ERR(1, 114, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyImport_ImportModule("cupy.cuda.function"); if (unlikely(!__pyx_t_1)) __PYX_ERR(6, 4, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_CPointer = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.function", "CPointer",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_CPointer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_CPointer),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_CPointer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_CPointer),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_CPointer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_CPointer),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_CPointer) __PYX_ERR(6, 4, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_Function = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.function", "Function",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_Function), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_Function),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_Function), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_Function),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_Function), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_Function),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_Function) __PYX_ERR(6, 9, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_8function_Function = (struct __pyx_vtabstruct_4cupy_4cuda_8function_Function*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_Function); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_8function_Function)) __PYX_ERR(6, 9, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_Module = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.function", "Module",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_Module), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_Module),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_Module), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_Module),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_Module), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_Module),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_Module) __PYX_ERR(6, 20, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_8function_Module = (struct __pyx_vtabstruct_4cupy_4cuda_8function_Module*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_Module); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_8function_Module)) __PYX_ERR(6, 20, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_LinkState = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy.cuda.function", "LinkState",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_LinkState), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_LinkState),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_LinkState), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_LinkState),
  #else
  sizeof(struct __pyx_obj_4cupy_4cuda_8function_LinkState), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_4cuda_8function_LinkState),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_LinkState) __PYX_ERR(6, 34, __pyx_L1_error)
  __pyx_vtabptr_4cupy_4cuda_8function_LinkState = (struct __pyx_vtabstruct_4cupy_4cuda_8function_LinkState*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_4cuda_8function_LinkState); if (unlikely(!__pyx_vtabptr_4cupy_4cuda_8function_LinkState)) __PYX_ERR(6, 34, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyImport_ImportModule("cupy._core._carray"); if (unlikely(!__pyx_t_1)) __PYX_ERR(7, 22, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_mdspan = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy._core._carray", "mdspan",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_mdspan), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_mdspan),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_mdspan), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_mdspan),
  #else
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_mdspan), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_mdspan),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_mdspan) __PYX_ERR(7, 22, __pyx_L1_error)
  __pyx_vtabptr_4cupy_5_core_7_carray_mdspan = (struct __pyx_vtabstruct_4cupy_5_core_7_carray_mdspan*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_mdspan); if (unlikely(!__pyx_vtabptr_4cupy_5_core_7_carray_mdspan)) __PYX_ERR(7, 22, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_CArray = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy._core._carray", "CArray",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_CArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_CArray),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_CArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_CArray),
  #else
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_CArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_CArray),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_CArray) __PYX_ERR(7, 31, __pyx_L1_error)
  __pyx_vtabptr_4cupy_5_core_7_carray_CArray = (struct __pyx_vtabstruct_4cupy_5_core_7_carray_CArray*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_CArray); if (unlikely(!__pyx_vtabptr_4cupy_5_core_7_carray_CArray)) __PYX_ERR(7, 31, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_CIndexer = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy._core._carray", "CIndexer",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer),
  #else
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_CIndexer),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_CIndexer) __PYX_ERR(7, 44, __pyx_L1_error)
  __pyx_vtabptr_4cupy_5_core_7_carray_CIndexer = (struct __pyx_vtabstruct_4cupy_5_core_7_carray_CIndexer*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_CIndexer); if (unlikely(!__pyx_vtabptr_4cupy_5_core_7_carray_CIndexer)) __PYX_ERR(7, 44, __pyx_L1_error)
  __pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_Indexer = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy._core._carray", "Indexer",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_Indexer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_Indexer),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_Indexer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_Indexer),
  #else
  sizeof(struct __pyx_obj_4cupy_5_core_7_carray_Indexer), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_7_carray_Indexer),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_Indexer) __PYX_ERR(7, 49, __pyx_L1_error)
  __pyx_vtabptr_4cupy_5_core_7_carray_Indexer = (struct __pyx_vtabstruct_4cupy_5_core_7_carray_Indexer*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_5_core_7_carray_Indexer); if (unlikely(!__pyx_vtabptr_4cupy_5_core_7_carray_Indexer)) __PYX_ERR(7, 49, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __pyx_t_1 = PyImport_ImportModule("cupy._core.core"); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 10, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_1);
  __pyx_mstate->__pyx_ptype_4cupy_5_core_4core__ndarray_base = __Pyx_ImportType_3_2_5(__pyx_t_1, "cupy._core.core", "_ndarray_base",
  #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000
  sizeof(struct __pyx_obj_4cupy_5_core_4core__ndarray_base), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_4core__ndarray_base),
  #elif CYTHON_COMPILING_IN_LIMITED_API
  sizeof(struct __pyx_obj_4cupy_5_core_4core__ndarray_base), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_4core__ndarray_base),
  #else
  sizeof(struct __pyx_obj_4cupy_5_core_4core__ndarray_base), __PYX_GET_STRUCT_ALIGNMENT_3_2_5(struct __pyx_obj_4cupy_5_core_4core__ndarray_base),
  #endif
  __Pyx_ImportType_CheckSize_Warn_3_2_5); if (!__pyx_mstate->__pyx_ptype_4cupy_5_core_4core__ndarray_base) __PYX_ERR(2, 10, __pyx_L1_error)
  __pyx_vtabptr_4cupy_5_core_4core__ndarray_base = (struct __pyx_vtabstruct_4cupy_5_core_4core__ndarray_base*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_4cupy_5_core_4core__ndarray_base); if (unlikely(!__pyx_vtabptr_4cupy_5_core_4core__ndarray_base)) __PYX_ERR(2, 10, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  __Pyx_RefNannyFinishContext();
  return 0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_RefNannyFinishContext();
  return -1;
}

static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0);
  /*--- Variable import code ---*/
  {
    __pyx_t_1 = PyImport_ImportModule("cupy_backends.cuda.api.runtime"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_int__is_hip_environment_deviceAt);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error)
    #endif
    const char * __pyx_import_name = __pyx_import_signature + 11;
    void **const __pyx_import_pointers[] = {(void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime__is_hip_environment, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMajor, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_deviceAttributeComputeCapabilityMinor, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorContextIsDestroyed, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorInvalidResourceHandle, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorInvalidValue, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorMemoryAllocation, (void **)&__pyx_vp_13cupy_backends_4cuda_3api_7runtime_errorPeerAccessAlreadyEnabled, (void **) NULL};
    void **const *__pyx_import_pointer = __pyx_import_pointers;
    const char *__pyx_import_current_signature = __pyx_import_signature;
    while (*__pyx_import_pointer) {
      if (__Pyx_ImportVoidPtr_3_2_5(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
      ++__pyx_import_pointer;
      __pyx_import_name = strchr(__pyx_import_name, '\0') + 1;
      __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1;
      if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature;
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  }
  __Pyx_RefNannyFinishContext();
  return 0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_RefNannyFinishContext();
  return -1;
}

static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  PyObject *__pyx_t_1 = NULL;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0);
  /*--- Function import code ---*/
  {
    __pyx_t_1 = PyImport_ImportModule("cupy_backends.cuda.api.runtime"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_int_int___pyx_skip_dispatch_runt);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error)
    #endif
    const char * __pyx_import_name = __pyx_import_signature + 30;
    void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_f_13cupy_backends_4cuda_3api_7runtime_runtimeGetVersion, (void (**)(void)) NULL};
    void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers;
    const char *__pyx_import_current_signature = __pyx_import_signature;
    while (*__pyx_import_pointer) {
      if (__Pyx_ImportFunction_3_2_5(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
      ++__pyx_import_pointer;
      __pyx_import_name = strchr(__pyx_import_name, '\0') + 1;
      __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1;
      if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature;
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  }
  {
    __pyx_t_1 = PyImport_ImportModule("cupy.cuda.device"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_PyObject_int___pyx_skip_dispatch);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error)
    #endif
    const char * __pyx_import_name = __pyx_import_signature + 66;
    void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_f_4cupy_4cuda_6device_get_compute_capability, (void (**)(void))&__pyx_f_4cupy_4cuda_6device_get_device_id, (void (**)(void)) NULL};
    void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers;
    const char *__pyx_import_current_signature = __pyx_import_signature;
    while (*__pyx_import_pointer) {
      if (__Pyx_ImportFunction_3_2_5(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
      ++__pyx_import_pointer;
      __pyx_import_name = strchr(__pyx_import_name, '\0') + 1;
      __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1;
      if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature;
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  }
  {
    __pyx_t_1 = PyImport_ImportModule("cupy._core.core"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_1);
    const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_struct___pyx_obj_4cupy_5_core_4c);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error)
    #endif
    const char * __pyx_import_name = __pyx_import_signature + 89;
    void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_f_4cupy_5_core_4core__convert_object_with_cuda_array_interface, (void (**)(void)) NULL};
    void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers;
    const char *__pyx_import_current_signature = __pyx_import_signature;
    while (*__pyx_import_pointer) {
      if (__Pyx_ImportFunction_3_2_5(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
      ++__pyx_import_pointer;
      __pyx_import_name = strchr(__pyx_import_name, '\0') + 1;
      __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1;
      if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature;
    }
    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
  }
  __Pyx_RefNannyFinishContext();
  return 0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_1);
  __Pyx_RefNannyFinishContext();
  return -1;
}

#if CYTHON_PEP489_MULTI_PHASE_INIT
static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/
static int __pyx_pymod_exec__callback(PyObject* module); /*proto*/
static PyModuleDef_Slot __pyx_moduledef_slots[] = {
  {Py_mod_create, (void*)__pyx_pymod_create},
  {Py_mod_exec, (void*)__pyx_pymod_exec__callback},
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
  {Py_mod_gil, __Pyx_FREETHREADING_COMPATIBLE},
  #endif
  #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE
  {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
  #endif
  {0, NULL}
};
#endif

#ifdef __cplusplus
namespace {
  struct PyModuleDef __pyx_moduledef =
  #else
  static struct PyModuleDef __pyx_moduledef =
  #endif
  {
      PyModuleDef_HEAD_INIT,
      "_callback",
      0, /* m_doc */
    #if CYTHON_USE_MODULE_STATE
      sizeof(__pyx_mstatetype), /* m_size */
    #else
      (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */
    #endif
      __pyx_methods /* m_methods */,
    #if CYTHON_PEP489_MULTI_PHASE_INIT
      __pyx_moduledef_slots, /* m_slots */
    #else
      NULL, /* m_reload */
    #endif
    #if CYTHON_USE_MODULE_STATE
      __pyx_m_traverse, /* m_traverse */
      __pyx_m_clear, /* m_clear */
      NULL /* m_free */
    #else
      NULL, /* m_traverse */
      NULL, /* m_clear */
      NULL /* m_free */
    #endif
  };
  #ifdef __cplusplus
} /* anonymous namespace */
#endif

/* PyModInitFuncType */
#ifndef CYTHON_NO_PYINIT_EXPORT
  #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC
#else
  #ifdef __cplusplus
  #define __Pyx_PyMODINIT_FUNC extern "C" PyObject *
  #else
  #define __Pyx_PyMODINIT_FUNC PyObject *
  #endif
#endif

__Pyx_PyMODINIT_FUNC PyInit__callback(void) CYTHON_SMALL_CODE; /*proto*/
__Pyx_PyMODINIT_FUNC PyInit__callback(void)
#if CYTHON_PEP489_MULTI_PHASE_INIT
{
  return PyModuleDef_Init(&__pyx_moduledef);
}
/* ModuleCreationPEP489 */
#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\
      || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000))
static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) {
    {
        PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think
        if (!module) {
            PyErr_Clear(); // just try the 3.8-3.12 version
            module = PyImport_ImportModule("_xxsubinterpreters");
            if (!module) goto bad;
        }
        PyObject *current = PyObject_CallMethod(module, "get_current", NULL);
        Py_DECREF(module);
        if (!current) goto bad;
        if (PyTuple_Check(current)) {
            PyObject *new_current = PySequence_GetItem(current, 0);
            Py_DECREF(current);
            current = new_current;
            if (!new_current) goto bad;
        }
        long long as_c_int = PyLong_AsLongLong(current);
        Py_DECREF(current);
        return as_c_int;
    }
  bad:
    PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n");
    return -1;
}
#endif
#if !CYTHON_USE_MODULE_STATE
static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) {
    static PY_INT64_T main_interpreter_id = -1;
#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000
    PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get());
#elif CYTHON_COMPILING_IN_GRAAL
    PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get());
#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\
      || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000))
    PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId();
#elif CYTHON_COMPILING_IN_LIMITED_API
    PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get());
#else
    PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp);
#endif
    if (unlikely(current_id == -1)) {
        return -1;
    }
    if (main_interpreter_id == -1) {
        main_interpreter_id = current_id;
        return 0;
    } else if (unlikely(main_interpreter_id != current_id)) {
        PyErr_SetString(
            PyExc_ImportError,
            "Interpreter change detected - this module can only be loaded into one interpreter per process.");
        return -1;
    }
    return 0;
}
#endif
static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none)
{
    PyObject *value = PyObject_GetAttrString(spec, from_name);
    int result = 0;
    if (likely(value)) {
        if (allow_none || value != Py_None) {
            result = PyDict_SetItemString(moddict, to_name, value);
        }
        Py_DECREF(value);
    } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
        PyErr_Clear();
    } else {
        result = -1;
    }
    return result;
}
static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) {
    PyObject *module = NULL, *moddict, *modname;
    CYTHON_UNUSED_VAR(def);
    #if !CYTHON_USE_MODULE_STATE
    if (__Pyx_check_single_interpreter())
        return NULL;
    #endif
    if (__pyx_m)
        return __Pyx_NewRef(__pyx_m);
    modname = PyObject_GetAttrString(spec, "name");
    if (unlikely(!modname)) goto bad;
    module = PyModule_NewObject(modname);
    Py_DECREF(modname);
    if (unlikely(!module)) goto bad;
    moddict = PyModule_GetDict(module);
    if (unlikely(!moddict)) goto bad;
    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad;
    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad;
    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad;
    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad;
    return module;
bad:
    Py_XDECREF(module);
    return NULL;
}


static CYTHON_SMALL_CODE int __pyx_pymod_exec__callback(PyObject *__pyx_pyinit_module)
#endif
{
  int stringtab_initialized = 0;
  #if CYTHON_USE_MODULE_STATE
  int pystate_addmodule_run = 0;
  #endif
  __pyx_mstatetype *__pyx_mstate = NULL;
  PyObject *__pyx_t_1 = NULL;
  PyObject *__pyx_t_2 = NULL;
  Py_ssize_t __pyx_t_3;
  PyObject *__pyx_t_4 = NULL;
  PyObject *__pyx_t_5 = NULL;
  PyObject *__pyx_t_6 = NULL;
  size_t __pyx_t_7;
  int __pyx_lineno = 0;
  const char *__pyx_filename = NULL;
  int __pyx_clineno = 0;
  __Pyx_RefNannyDeclarations
  #if CYTHON_PEP489_MULTI_PHASE_INIT
  if (__pyx_m) {
    if (__pyx_m == __pyx_pyinit_module) return 0;
    PyErr_SetString(PyExc_RuntimeError, "Module '_callback' has already been imported. Re-initialisation is not supported.");
    return -1;
  }
  #else
  if (__pyx_m) return __Pyx_NewRef(__pyx_m);
  #endif
  /*--- Module creation code ---*/
  #if CYTHON_PEP489_MULTI_PHASE_INIT
  __pyx_t_1 = __pyx_pyinit_module;
  Py_INCREF(__pyx_t_1);
  #else
  __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error)
  #endif
  #if CYTHON_USE_MODULE_STATE
  {
    int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef);
    __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "_callback" pseudovariable */
    if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
    pystate_addmodule_run = 1;
  }
  #else
  __pyx_m = __pyx_t_1;
  #endif
  #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
  PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED);
  #endif
  __pyx_mstate = __pyx_mstate_global;
  CYTHON_UNUSED_VAR(__pyx_t_1);
  __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error)
  Py_INCREF(__pyx_mstate->__pyx_d);
  __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error)
  __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error)
  if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
  /* ImportRefnannyAPI */
  #if CYTHON_REFNANNY
  __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny");
  if (!__Pyx_RefNanny) {
    PyErr_Clear();
    __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny");
    if (!__Pyx_RefNanny)
        Py_FatalError("failed to import 'refnanny' module");
  }
  #endif
  
__Pyx_RefNannySetupContext("PyInit__callback", 0);
  __Pyx_init_runtime_version();
  if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error)
  __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error)
  __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error)
  /*--- Library function declarations ---*/
  /*--- Initialize various global constants etc. ---*/
  if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  stringtab_initialized = 1;
  if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  if (__pyx_module_is_main_cupy__fft___callback) {
    if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  }
  {
    PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error)
    if (!PyDict_GetItemString(modules, "cupy.fft._callback")) {
      if (unlikely((PyDict_SetItemString(modules, "cupy.fft._callback", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
    }
  }
  /*--- Builtin init code ---*/
  if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  /*--- Constants init code ---*/
  if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  /*--- Global type/function init code ---*/
  (void)__Pyx_modinit_global_init_code(__pyx_mstate);
  (void)__Pyx_modinit_variable_export_code(__pyx_mstate);
  (void)__Pyx_modinit_function_export_code(__pyx_mstate);
  if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
  if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
  if (unlikely((__Pyx_modinit_variable_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
  if (unlikely((__Pyx_modinit_function_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
  /*--- Execution code ---*/

  /* "cupy/fft/_callback.pyx":8
 * from cupy.cuda.memory cimport MemoryPointer
 * 
 * import hashlib             # <<<<<<<<<<<<<<
 * import importlib
 * import os
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_hashlib, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 8, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_hashlib, __pyx_t_2) < (0)) __PYX_ERR(0, 8, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":9
 * 
 * import hashlib
 * import importlib             # <<<<<<<<<<<<<<
 * import os
 * import re
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_importlib, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 9, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_importlib, __pyx_t_2) < (0)) __PYX_ERR(0, 9, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":10
 * import hashlib
 * import importlib
 * import os             # <<<<<<<<<<<<<<
 * import re
 * import shutil
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_os, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_os, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":11
 * import importlib
 * import os
 * import re             # <<<<<<<<<<<<<<
 * import shutil
 * import string
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_re, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_re, __pyx_t_2) < (0)) __PYX_ERR(0, 11, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":12
 * import os
 * import re
 * import shutil             # <<<<<<<<<<<<<<
 * import string
 * import subprocess
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_shutil, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 12, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_shutil, __pyx_t_2) < (0)) __PYX_ERR(0, 12, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":13
 * import re
 * import shutil
 * import string             # <<<<<<<<<<<<<<
 * import subprocess
 * import sys
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_string, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 13, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_string, __pyx_t_2) < (0)) __PYX_ERR(0, 13, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":14
 * import shutil
 * import string
 * import subprocess             # <<<<<<<<<<<<<<
 * import sys
 * import sysconfig
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_subprocess, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 14, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_subprocess, __pyx_t_2) < (0)) __PYX_ERR(0, 14, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":15
 * import string
 * import subprocess
 * import sys             # <<<<<<<<<<<<<<
 * import sysconfig
 * import tempfile
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_sys, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 15, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_sys, __pyx_t_2) < (0)) __PYX_ERR(0, 15, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":16
 * import subprocess
 * import sys
 * import sysconfig             # <<<<<<<<<<<<<<
 * import tempfile
 * import threading
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_sysconfig, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 16, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_sysconfig, __pyx_t_2) < (0)) __PYX_ERR(0, 16, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":17
 * import sys
 * import sysconfig
 * import tempfile             # <<<<<<<<<<<<<<
 * import threading
 * import warnings
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_tempfile, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 17, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_tempfile, __pyx_t_2) < (0)) __PYX_ERR(0, 17, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":18
 * import sysconfig
 * import tempfile
 * import threading             # <<<<<<<<<<<<<<
 * import warnings
 * 
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_threading, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 18, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_threading, __pyx_t_2) < (0)) __PYX_ERR(0, 18, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":19
 * import tempfile
 * import threading
 * import warnings             # <<<<<<<<<<<<<<
 * 
 * from cuda.pathfinder import find_nvidia_binary_utility
*/
  __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_warnings, 0, 0, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 19, __pyx_L1_error)
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_warnings, __pyx_t_2) < (0)) __PYX_ERR(0, 19, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":21
 * import warnings
 * 
 * from cuda.pathfinder import find_nvidia_binary_utility             # <<<<<<<<<<<<<<
 * from cuda.pathfinder import find_nvidia_header_directory
 * 
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_find_nvidia_binary_utility};
    __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cuda_pathfinder, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_find_nvidia_binary_utility};
    __pyx_t_3 = 0; {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_2, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 21, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_3], __pyx_t_4) < (0)) __PYX_ERR(0, 21, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":22
 * 
 * from cuda.pathfinder import find_nvidia_binary_utility
 * from cuda.pathfinder import find_nvidia_header_directory             # <<<<<<<<<<<<<<
 * 
 * from cupy import __version__ as _cupy_ver
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_find_nvidia_header_directory};
    __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cuda_pathfinder, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_find_nvidia_header_directory};
    __pyx_t_3 = 0; {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_2, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 22, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_3], __pyx_t_4) < (0)) __PYX_ERR(0, 22, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":24
 * from cuda.pathfinder import find_nvidia_header_directory
 * 
 * from cupy import __version__ as _cupy_ver             # <<<<<<<<<<<<<<
 * from cupy._environment import (get_nvcc_path, get_cuda_path)
 * from cupy.cuda.compiler import (_get_bool_env_variable, CompileException)
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_version};
    __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 24, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_version};
    __pyx_t_3 = 0; {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_2, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 24, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      switch (__pyx_t_3) {
        case 0:
        if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_cupy_ver, __pyx_t_4) < (0)) __PYX_ERR(0, 24, __pyx_L1_error)
        break;
        default:;
      }
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":25
 * 
 * from cupy import __version__ as _cupy_ver
 * from cupy._environment import (get_nvcc_path, get_cuda_path)             # <<<<<<<<<<<<<<
 * from cupy.cuda.compiler import (_get_bool_env_variable, CompileException)
 * from cupy.cuda.compiler import _compile_with_cache_cuda
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_get_nvcc_path,__pyx_mstate_global->__pyx_n_u_get_cuda_path};
    __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy__environment, __pyx_imported_names, 2, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 25, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_get_nvcc_path,__pyx_mstate_global->__pyx_n_u_get_cuda_path};
    for (__pyx_t_3=0; __pyx_t_3 < 2; __pyx_t_3++) {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_2, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 25, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_3], __pyx_t_4) < (0)) __PYX_ERR(0, 25, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":26
 * from cupy import __version__ as _cupy_ver
 * from cupy._environment import (get_nvcc_path, get_cuda_path)
 * from cupy.cuda.compiler import (_get_bool_env_variable, CompileException)             # <<<<<<<<<<<<<<
 * from cupy.cuda.compiler import _compile_with_cache_cuda
 * 
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_get_bool_env_variable,__pyx_mstate_global->__pyx_n_u_CompileException};
    __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda_compiler, __pyx_imported_names, 2, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 26, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_get_bool_env_variable,__pyx_mstate_global->__pyx_n_u_CompileException};
    for (__pyx_t_3=0; __pyx_t_3 < 2; __pyx_t_3++) {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_2, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 26, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_3], __pyx_t_4) < (0)) __PYX_ERR(0, 26, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":27
 * from cupy._environment import (get_nvcc_path, get_cuda_path)
 * from cupy.cuda.compiler import (_get_bool_env_variable, CompileException)
 * from cupy.cuda.compiler import _compile_with_cache_cuda             # <<<<<<<<<<<<<<
 * 
 * 
*/
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_compile_with_cache_cuda};
    __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cupy_cuda_compiler, __pyx_imported_names, 1, NULL, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 27, __pyx_L1_error)
  }
  __pyx_t_2 = __pyx_t_1;
  __Pyx_GOTREF(__pyx_t_2);
  {
    PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_compile_with_cache_cuda};
    __pyx_t_3 = 0; {
      __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_2, __pyx_imported_names[__pyx_t_3]); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 27, __pyx_L1_error)
      __Pyx_GOTREF(__pyx_t_4);
      if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_3], __pyx_t_4) < (0)) __PYX_ERR(0, 27, __pyx_L1_error)
      __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    }
  }
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":31
 * 
 * # information needed for building an external module
 * cdef list _cc = []             # <<<<<<<<<<<<<<
 * cdef str _python_include = None
 * cdef list _nvcc = []
*/
  __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 31, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cc);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cc, ((PyObject*)__pyx_t_2));
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":32
 * # information needed for building an external module
 * cdef list _cc = []
 * cdef str _python_include = None             # <<<<<<<<<<<<<<
 * cdef list _nvcc = []
 * cdef str _cuda_path = None
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__python_include);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__python_include, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":33
 * cdef list _cc = []
 * cdef str _python_include = None
 * cdef list _nvcc = []             # <<<<<<<<<<<<<<
 * cdef str _cuda_path = None
 * cdef str _cuda_include = None
*/
  __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 33, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvcc);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvcc, ((PyObject*)__pyx_t_2));
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":34
 * cdef str _python_include = None
 * cdef list _nvcc = []
 * cdef str _cuda_path = None             # <<<<<<<<<<<<<<
 * cdef str _cuda_include = None
 * cdef str _nvprune = None
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cuda_path);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cuda_path, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":35
 * cdef list _nvcc = []
 * cdef str _cuda_path = None
 * cdef str _cuda_include = None             # <<<<<<<<<<<<<<
 * cdef str _nvprune = None
 * cdef str _build_ver = None
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cuda_include);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cuda_include, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":36
 * cdef str _cuda_path = None
 * cdef str _cuda_include = None
 * cdef str _nvprune = None             # <<<<<<<<<<<<<<
 * cdef str _build_ver = None
 * cdef int _cufft_ver = 0
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__nvprune);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__nvprune, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":37
 * cdef str _cuda_include = None
 * cdef str _nvprune = None
 * cdef str _build_ver = None             # <<<<<<<<<<<<<<
 * cdef int _cufft_ver = 0
 * cdef str _cupy_root = None
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__build_ver);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__build_ver, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":38
 * cdef str _nvprune = None
 * cdef str _build_ver = None
 * cdef int _cufft_ver = 0             # <<<<<<<<<<<<<<
 * cdef str _cupy_root = None
 * cdef str _cupy_include = None
*/
  __pyx_v_4cupy_3fft_9_callback__cufft_ver = 0;

  /* "cupy/fft/_callback.pyx":39
 * cdef str _build_ver = None
 * cdef int _cufft_ver = 0
 * cdef str _cupy_root = None             # <<<<<<<<<<<<<<
 * cdef str _cupy_include = None
 * cdef str _source_dir = None
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cupy_root);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cupy_root, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":40
 * cdef int _cufft_ver = 0
 * cdef str _cupy_root = None
 * cdef str _cupy_include = None             # <<<<<<<<<<<<<<
 * cdef str _source_dir = None
 * cdef str _ext_suffix = None
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cupy_include);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cupy_include, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":41
 * cdef str _cupy_root = None
 * cdef str _cupy_include = None
 * cdef str _source_dir = None             # <<<<<<<<<<<<<<
 * cdef str _ext_suffix = None
 * 
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__source_dir);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__source_dir, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":42
 * cdef str _cupy_include = None
 * cdef str _source_dir = None
 * cdef str _ext_suffix = None             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__ext_suffix);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__ext_suffix, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":46
 * 
 * # callback related stuff
 * cdef bint _is_init = False             # <<<<<<<<<<<<<<
 * cdef str _callback_dev_code = None
 * cdef str _callback_cache_dir = None
*/
  __pyx_v_4cupy_3fft_9_callback__is_init = 0;

  /* "cupy/fft/_callback.pyx":47
 * # callback related stuff
 * cdef bint _is_init = False
 * cdef str _callback_dev_code = None             # <<<<<<<<<<<<<<
 * cdef str _callback_cache_dir = None
 * cdef dict _callback_mgr = {}  # keep the Python modules alive
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__callback_dev_code);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__callback_dev_code, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":48
 * cdef bint _is_init = False
 * cdef str _callback_dev_code = None
 * cdef str _callback_cache_dir = None             # <<<<<<<<<<<<<<
 * cdef dict _callback_mgr = {}  # keep the Python modules alive
 * cdef object _callback_thread_local = threading.local()
*/
  __Pyx_INCREF(Py_None);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__callback_cache_dir);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__callback_cache_dir, ((PyObject*)Py_None));
  __Pyx_GIVEREF(Py_None);

  /* "cupy/fft/_callback.pyx":49
 * cdef str _callback_dev_code = None
 * cdef str _callback_cache_dir = None
 * cdef dict _callback_mgr = {}  # keep the Python modules alive             # <<<<<<<<<<<<<<
 * cdef object _callback_thread_local = threading.local()
 * 
*/
  __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 49, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__callback_mgr);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__callback_mgr, ((PyObject*)__pyx_t_2));
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":50
 * cdef str _callback_cache_dir = None
 * cdef dict _callback_mgr = {}  # keep the Python modules alive
 * cdef object _callback_thread_local = threading.local()             # <<<<<<<<<<<<<<
 * 
 * 
*/
  __pyx_t_4 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_threading); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 50, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_local); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 50, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
  __pyx_t_7 = 1;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL};
    __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_7, (1-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 50, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__callback_thread_local);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__callback_thread_local, __pyx_t_2);
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":217
 * # specializations.
 * cdef dict _cc_major_map = {
 *     '12': ('120', '121'),             # <<<<<<<<<<<<<<
 *     '10': ('100', '103'),
 *     '9': ('90',),
*/
  __pyx_t_2 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 217, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_t_2, __pyx_mstate_global->__pyx_kp_u_12, __pyx_mstate_global->__pyx_tuple[5]) < (0)) __PYX_ERR(0, 217, __pyx_L1_error)

  /* "cupy/fft/_callback.pyx":218
 * cdef dict _cc_major_map = {
 *     '12': ('120', '121'),
 *     '10': ('100', '103'),             # <<<<<<<<<<<<<<
 *     '9': ('90',),
 *     '8': ('80', '86', '87'),
*/
  if (PyDict_SetItem(__pyx_t_2, __pyx_mstate_global->__pyx_kp_u_10, __pyx_mstate_global->__pyx_tuple[6]) < (0)) __PYX_ERR(0, 217, __pyx_L1_error)

  /* "cupy/fft/_callback.pyx":219
 *     '12': ('120', '121'),
 *     '10': ('100', '103'),
 *     '9': ('90',),             # <<<<<<<<<<<<<<
 *     '8': ('80', '86', '87'),
 * }
*/
  if (PyDict_SetItem(__pyx_t_2, __pyx_mstate_global->__pyx_kp_u_9, __pyx_mstate_global->__pyx_tuple[7]) < (0)) __PYX_ERR(0, 217, __pyx_L1_error)

  /* "cupy/fft/_callback.pyx":220
 *     '10': ('100', '103'),
 *     '9': ('90',),
 *     '8': ('80', '86', '87'),             # <<<<<<<<<<<<<<
 * }
 * 
*/
  if (PyDict_SetItem(__pyx_t_2, __pyx_mstate_global->__pyx_kp_u_8, __pyx_mstate_global->__pyx_tuple[8]) < (0)) __PYX_ERR(0, 217, __pyx_L1_error)
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback__cc_major_map);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback__cc_major_map, ((PyObject*)__pyx_t_2));
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":327
 * 
 * # make it a plain Python function so that we can mock-test it
 * def get_cache_dir():             # <<<<<<<<<<<<<<
 *     global _callback_cache_dir
 *     if _callback_cache_dir is None:
*/
  __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_4cupy_3fft_9_callback_1get_cache_dir, NULL, __pyx_mstate_global->__pyx_n_u_cupy_fft__callback); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 327, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_get_cache_dir, __pyx_t_2) < (0)) __PYX_ERR(0, 327, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":691
 * # LHS is our legacy convention
 * # RHS is a valid C identifier pattern
 * cdef object cupy_load_callback_pattern = re.compile(             # <<<<<<<<<<<<<<
 *     r'd_loadCallbackPtr\s*=\s*([A-Za-z_]\w*)\s*;')
 * 
*/
  __pyx_t_6 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_re); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 691, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_compile); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 691, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
  __pyx_t_7 = 1;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_mstate_global->__pyx_kp_u_d_loadCallbackPtr_s_s_A_Za_z__w};
    __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 691, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback_cupy_load_callback_pattern);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback_cupy_load_callback_pattern, __pyx_t_2);
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":694
 *     r'd_loadCallbackPtr\s*=\s*([A-Za-z_]\w*)\s*;')
 * 
 * cdef object cupy_store_callback_pattern = re.compile(             # <<<<<<<<<<<<<<
 *     r'd_storeCallbackPtr\s*=\s*([A-Za-z_]\w*)\s*;')
 * 
*/
  __pyx_t_5 = NULL;
  __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_re); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 694, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_compile); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 694, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_t_7 = 1;
  {
    PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_d_storeCallbackPtr_s_s_A_Za_z__w};
    __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
    __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
    if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 694, __pyx_L1_error)
    __Pyx_GOTREF(__pyx_t_2);
  }
  __Pyx_XGOTREF(__pyx_v_4cupy_3fft_9_callback_cupy_store_callback_pattern);
  __Pyx_DECREF_SET(__pyx_v_4cupy_3fft_9_callback_cupy_store_callback_pattern, __pyx_t_2);
  __Pyx_GIVEREF(__pyx_t_2);
  __pyx_t_2 = 0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__ThreadLocal(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x8b3d916, 0xf743b78, 0xe63718f, b'_current_cufft_callback')
*/
  __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_4cupy_3fft_9_callback_5__pyx_unpickle__ThreadLocal, NULL, __pyx_mstate_global->__pyx_n_u_cupy_fft__callback); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__ThreadLocal, __pyx_t_2) < (0)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":1
 * cdef extern from *:             # <<<<<<<<<<<<<<
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
*/
  __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_4cupy_3fft_9_callback_7__pyx_unpickle__CallbackManager, NULL, __pyx_mstate_global->__pyx_n_u_cupy_fft__callback); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__CallbackManager, __pyx_t_2) < (0)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle__LegacyCallbackManager(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x3241615, 0x86aada4, 0x3c579aa, b'cb_load, cb_load_data, cb_store, cb_store_data, identity, mod')
*/
  __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_4cupy_3fft_9_callback_9__pyx_unpickle__LegacyCallbackManager, NULL, __pyx_mstate_global->__pyx_n_u_cupy_fft__callback); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__LegacyCallbackMa, __pyx_t_2) < (0)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":1
 * cdef extern from *:             # <<<<<<<<<<<<<<
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
*/
  __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_4cupy_3fft_9_callback_11__pyx_unpickle__JITCallbackManager, NULL, __pyx_mstate_global->__pyx_n_u_cupy_fft__callback); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle__JITCallbackManag, __pyx_t_2) < (0)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "(tree fragment)":4
 *     int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
 *     int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
 * def __pyx_unpickle_set_cufft_callbacks(__pyx_type, long __pyx_checksum, tuple __pyx_state):             # <<<<<<<<<<<<<<
 *     cdef object __pyx_result
 *     __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xf6c65f0, 0x8412ee4, 0x715cae0, b'mgr, mgr_prev')
*/
  __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_4cupy_3fft_9_callback_13__pyx_unpickle_set_cufft_callbacks, NULL, __pyx_mstate_global->__pyx_n_u_cupy_fft__callback); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_set_cufft_callbac, __pyx_t_2) < (0)) __PYX_ERR(3, 4, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /* "cupy/fft/_callback.pyx":1
 * from libc.stdint cimport intptr_t             # <<<<<<<<<<<<<<
 * 
 * from cupy_backends.cuda.api cimport runtime
*/
  __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_2);
  if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;

  /*--- Wrapped vars code ---*/

  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_XDECREF(__pyx_t_2);
  __Pyx_XDECREF(__pyx_t_4);
  __Pyx_XDECREF(__pyx_t_5);
  __Pyx_XDECREF(__pyx_t_6);
  if (__pyx_m) {
    if (__pyx_mstate->__pyx_d && stringtab_initialized) {
      __Pyx_AddTraceback("init cupy.fft._callback", __pyx_clineno, __pyx_lineno, __pyx_filename);
    }
    #if !CYTHON_USE_MODULE_STATE
    Py_CLEAR(__pyx_m);
    #else
    Py_DECREF(__pyx_m);
    if (pystate_addmodule_run) {
      PyObject *tp, *value, *tb;
      PyErr_Fetch(&tp, &value, &tb);
      PyState_RemoveModule(&__pyx_moduledef);
      PyErr_Restore(tp, value, tb);
    }
    #endif
  } else if (!PyErr_Occurred()) {
    PyErr_SetString(PyExc_ImportError, "init cupy.fft._callback");
  }
  __pyx_L0:;
  __Pyx_RefNannyFinishContext();
  #if CYTHON_PEP489_MULTI_PHASE_INIT
  return (__pyx_m != NULL) ? 0 : -1;
  #else
  return __pyx_m;
  #endif
}
/* #### Code section: pystring_table ### */
/* #### Code section: cached_builtins ### */

static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) {
  CYTHON_UNUSED_VAR(__pyx_mstate);
  __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(0, 138, __pyx_L1_error)
  __pyx_builtin_open = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_open); if (!__pyx_builtin_open) __PYX_ERR(0, 267, __pyx_L1_error)

  /* Cached unbound methods */
  __pyx_mstate->__pyx_umethod_PyDict_Type_get.type = (PyObject*)&PyDict_Type;
  __pyx_mstate->__pyx_umethod_PyDict_Type_get.method_name = &__pyx_mstate->__pyx_n_u_get;
  __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type;
  __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items;
  __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type;
  __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop;
  __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type;
  __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values;
  return 0;
  __pyx_L1_error:;
  return -1;
}
/* #### Code section: cached_constants ### */

static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) {
  __Pyx_RefNannyDeclarations
  CYTHON_UNUSED_VAR(__pyx_mstate);
  __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0);

  /* "cupy/fft/_callback.pyx":105
 *     if _cufft_ver < 12000:
 *         _cc_major_map = {**_cc_major_map, **{
 *             '7': ('70', '72', '75'),             # <<<<<<<<<<<<<<
 *             '6': ('60', '61', '62'),
 *             '5': ('50', '52', '53'),
*/
  __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(3, __pyx_mstate_global->__pyx_kp_u_70, __pyx_mstate_global->__pyx_kp_u_72, __pyx_mstate_global->__pyx_kp_u_75); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 105, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]);

  /* "cupy/fft/_callback.pyx":106
 *         _cc_major_map = {**_cc_major_map, **{
 *             '7': ('70', '72', '75'),
 *             '6': ('60', '61', '62'),             # <<<<<<<<<<<<<<
 *             '5': ('50', '52', '53'),
 *         }}
*/
  __pyx_mstate_global->__pyx_tuple[1] = PyTuple_Pack(3, __pyx_mstate_global->__pyx_kp_u_60, __pyx_mstate_global->__pyx_kp_u_61, __pyx_mstate_global->__pyx_kp_u_62); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(0, 106, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]);

  /* "cupy/fft/_callback.pyx":107
 *             '7': ('70', '72', '75'),
 *             '6': ('60', '61', '62'),
 *             '5': ('50', '52', '53'),             # <<<<<<<<<<<<<<
 *         }}
 *     _ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
*/
  __pyx_mstate_global->__pyx_tuple[2] = PyTuple_Pack(3, __pyx_mstate_global->__pyx_kp_u_50, __pyx_mstate_global->__pyx_kp_u_52, __pyx_mstate_global->__pyx_kp_u_53); if (unlikely(!__pyx_mstate_global->__pyx_tuple[2])) __PYX_ERR(0, 107, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[2]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[2]);

  /* "cupy/fft/_callback.pyx":267
 * 
 *     if _callback_dev_code is None:
 *         with open(os.path.join(_source_dir, 'cupy_cufftXt.cu')) as f:             # <<<<<<<<<<<<<<
 *             support = _callback_dev_code = f.read()
 *     else:
*/
  __pyx_mstate_global->__pyx_tuple[3] = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[3])) __PYX_ERR(0, 267, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[3]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[3]);

  /* "cupy/fft/_callback.pyx":295
 *             os.path.join(tempdir, 'cupy_cufftXt.cu'),
 *             cmd[1:], 'nvcc')
 *         dump = _get_bool_env_variable(             # <<<<<<<<<<<<<<
 *             'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
 *         if dump:
*/
  __pyx_mstate_global->__pyx_tuple[4] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_n_u_CUPY_DUMP_CUDA_SOURCE_ON_ERROR, Py_False); if (unlikely(!__pyx_mstate_global->__pyx_tuple[4])) __PYX_ERR(0, 295, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[4]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[4]);

  /* "cupy/fft/_callback.pyx":217
 * # specializations.
 * cdef dict _cc_major_map = {
 *     '12': ('120', '121'),             # <<<<<<<<<<<<<<
 *     '10': ('100', '103'),
 *     '9': ('90',),
*/
  __pyx_mstate_global->__pyx_tuple[5] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_kp_u_120, __pyx_mstate_global->__pyx_kp_u_121); if (unlikely(!__pyx_mstate_global->__pyx_tuple[5])) __PYX_ERR(0, 217, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[5]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[5]);

  /* "cupy/fft/_callback.pyx":218
 * cdef dict _cc_major_map = {
 *     '12': ('120', '121'),
 *     '10': ('100', '103'),             # <<<<<<<<<<<<<<
 *     '9': ('90',),
 *     '8': ('80', '86', '87'),
*/
  __pyx_mstate_global->__pyx_tuple[6] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_kp_u_100, __pyx_mstate_global->__pyx_kp_u_103); if (unlikely(!__pyx_mstate_global->__pyx_tuple[6])) __PYX_ERR(0, 218, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[6]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[6]);

  /* "cupy/fft/_callback.pyx":219
 *     '12': ('120', '121'),
 *     '10': ('100', '103'),
 *     '9': ('90',),             # <<<<<<<<<<<<<<
 *     '8': ('80', '86', '87'),
 * }
*/
  __pyx_mstate_global->__pyx_tuple[7] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_kp_u_90); if (unlikely(!__pyx_mstate_global->__pyx_tuple[7])) __PYX_ERR(0, 219, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[7]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[7]);

  /* "cupy/fft/_callback.pyx":220
 *     '10': ('100', '103'),
 *     '9': ('90',),
 *     '8': ('80', '86', '87'),             # <<<<<<<<<<<<<<
 * }
 * 
*/
  __pyx_mstate_global->__pyx_tuple[8] = PyTuple_Pack(3, __pyx_mstate_global->__pyx_kp_u_80, __pyx_mstate_global->__pyx_kp_u_86, __pyx_mstate_global->__pyx_kp_u_87); if (unlikely(!__pyx_mstate_global->__pyx_tuple[8])) __PYX_ERR(0, 220, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[8]);
  __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[8]);
  #if CYTHON_IMMORTAL_CONSTANTS
  {
    PyObject **table = __pyx_mstate->__pyx_tuple;
    for (Py_ssize_t i=0; i<9; ++i) {
      #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
      #if PY_VERSION_HEX < 0x030E0000
      if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1)
      #else
      if (PyUnstable_Object_IsUniquelyReferenced(table[i]))
      #endif
      {
        Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL);
      }
      #else
      Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT);
      #endif
    }
  }
  #endif
  __Pyx_RefNannyFinishContext();
  return 0;
  __pyx_L1_error:;
  __Pyx_RefNannyFinishContext();
  return -1;
}
/* #### Code section: init_constants ### */

static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) {
  CYTHON_UNUSED_VAR(__pyx_mstate);
  {
    const struct { const unsigned int length: 8; } index[] = {{2},{2},{3},{3},{2},{3},{3},{2},{1},{2},{2},{2},{1},{2},{2},{2},{1},{2},{2},{2},{1},{2},{2},{2},{1},{2},{10},{18},{22},{18},{19},{20},{2},{28},{2},{51},{2},{4},{179},{3},{9},{93},{10},{8},{1},{1},{2},{2},{0},{1},{2},{81},{8},{9},{11},{2},{61},{62},{32},{6},{9},{17},{13},{7},{4},{43},{33},{9},{9},{22},{15},{14},{12},{42},{43},{3},{6},{7},{6},{5},{2},{29},{6},{30},{33},{9},{2},{14},{9},{23},{26},{9},{44},{46},{51},{30},{23},{20},{2},{2},{45},{47},{205},{4},{4},{26},{7},{10},{11},{27},{71},{9},{9},{19},{26},{16},{23},{19},{26},{16},{23},{9},{9},{9},{9},{14},{30},{3},{16},{18},{16},{10},{19},{22},{4},{20},{8},{18},{12},{7},{15},{12},{12},{8},{16},{13},{13},{6},{16},{7},{18},{7},{24},{8},{6},{11},{4},{24},{15},{6},{5},{13},{4},{17},{14},{9},{18},{15},{18},{20},{9},{3},{6},{17},{18},{6},{21},{22},{8},{3},{7},{4},{6},{9},{9},{3},{7},{8},{9},{11},{8},{8},{10},{8},{8},{26},{28},{3},{10},{22},{13},{14},{13},{13},{8},{12},{5},{6},{7},{9},{9},{7},{5},{6},{5},{3},{4},{6},{3},{5},{6},{5},{8},{8},{7},{3},{10},{16},{7},{4},{8},{7},{4},{7},{13},{4},{2},{4},{4},{9},{8},{3},{13},{14},{11},{10},{31},{34},{37},{27},{34},{14},{12},{2},{4},{10},{17},{13},{6},{3},{6},{17},{11},{14},{13},{19},{12},{10},{12},{19},{6},{3},{23},{5},{10},{6},{6},{6},{10},{10},{3},{9},{8},{8},{9},{3},{8},{9},{6},{15},{4},{6},{11},{1},{4},{8},{5},{102},{227},{47},{130}};
    #if (CYTHON_COMPRESS_STRINGS) == 3 && __PYX_LIMITED_VERSION_HEX >= 0x030e0000 /* compression: zstd (2079 bytes) */
const char* const cstring = "(\265/\375`\343\020\255@\000\372\\\240\020-\300\260j\033\360\027\034$\t\354\256W.S\223]\267'\352\177Q*\004\341\221@P9\177\343\030Q\256\232h\312\245_Q\024\205m\0338\001\020\001\374\000\377\000Q\233S{\376\360\265\316\366\311\005\341\344:v\030[\325\330f\237o,\005\335\353^#\000\344\362@\353\2179X\0167\273E`\003Ya\344g\037!d\004\326\257/\270\323;\223\013\327\214\325\222*\013\362\225\325/\310\377xFK\212\356\354\265\310\277m\307\366r\220-\n\014\212\007C\301,\330\373\215\373}Uo\237\365F\374\326-\347\005\343\364\225d=\233~\355G\n\013#\005\313\363\200\001A&\250\330\322\232m\3716\317\r\333\353;J9}y4\026\275\307\332\322T\"T\036\310{\035\333i-\007YZ\361\227\333\307\314\001\205\222\271H\220D\032X,\021\211N\"\027\213jD L\036\tL\005\302\316\326dH*\232\210FC5\016\034dP\270T42\224\n\032\231lH\324\002fC\322\370\305\210\252\274\301#\221 \301\005\206\005\206\004\006F\305\005\344B\342\342\002b1a!aa\201\240\002R!QQ\201\340\221\200L@&\020HLHL@\006\002\317\273\034\333\236^\276\007.\2077\014\212\240\260\360\330\\\337\363\036\347\354\033\235\344i\271-\260:6\310\313\331\264\326\307\315\202\320\375\371\300\335<O\333H*\030\351\201\353\354\"\304Y\247\315=n\201\201\003\037\0314(\207\207]x\"\030\nd\034\360\014\375\030\211\233\236Cwj\254iD%g\203\":t*\327\221\320K\322l\032\227\365\034-\n\213\325\231\220-\217\031\207\301\270\266\331r)\033\344\363\311u\3635\310\021\333\261\351\363\254\213\r\002\362\352\270\004\010\360I\255\355\370^\275\347\306Ovv=\257J\342\266\262{%\310k\243\325\326R\214\305\327\236\205\263%\007F@\216\224\362\214\2515{V\235c\034\034\222\265/b\334\214\263\036.\005c3\007.\234QZ\037\341z\267\036\036$\2131\210A\000\005\020\264\240^\017\310\217\343\351},\251\3308\036\307\366\035\362\022\360\334! \243\227\243\001L~\324w~\323EJ\333\316\3078\332:K\315U]IM\327\337\330\251\366z\254k\356C\365w\243\220B\372\335;=\223\3206&\365\276*\324g\333M\252dl\314M\335\306\372\025\262.\344\327\344\204l_\203=\177[=\036k\321\242\321\252c\263\216\373\354Ij5\322f\276\263\024\364\033O:\025F\225\202\251`,\252\251DP""\216\212\206j8(r[\207\"\351\330Mq\337\026\033\343\266w\303 \233\240\034\034\222\314%\223\241(\205\213R$\231\215\213\006f\302\306$S\231(\005\202$2\201\\\270\000\231h4,\021J\206bi\260q\231(\007\305\202\231p\221\350\304\242\r2\022\210d)\346R\254\340N\037\317\305\304\303C\313p\357\217\333f\223?\313\007\364\374)-\346\226\023\251\355\367|n\232\035\200\200\234\343\257\2129\270 \353f\332\231\214=s\021\212\217\363\372\220Vk\0348}\214 z\247Q\177A\254\263\250\202B;\242\330\346\203\245T/\322\"\275\213\251:R;\241\002\204m=\243o\3061\254\270\255b\277\357\304+\337\331\207\013\271c\007\233\020q\377G$O\352\254\000\034\3536\216\230d\r\200\201\251\261S\311\036\353!\0278\260\242K\205\331\017\330\354\343C9\325s\317\266\356\326\347>sr6\307\177\032\337\337\377{\345\373\363\371]F\3237\220{F\367o~\222r\235\374\337d\322\272H\374\031Y\357l\233n\373\261\271\3419\356\321X\302IM\227\252\217mK\331Q\212\222G\2702z\324\366\315\201\372\261\252\256Z\341\250\345\300\336a\2247\273\365\200\314(\325\327\373 \307\216\347\016\247\366\331\331\231\342\367\335\260\353\354\024\361\233\234M\373&9c\257nX\202Q_\214\021\214\354\275\025\234V\317\217\177*c\247\332\353\323\301\201\326\250q\235CF\231\031\021\021I\222$\303\001!\010\303\221\240s\022\362\001\302\361X\251,CB\023\310\210\024$\005)\225\332\ru+\236\317\370\270\363\343X\007^\033\327\2539\244\356\026\252\204\004\356 qk4\267\206\014\215\253`\312\247\002\346\265\001(\231\260\304\376KU{y\345`k\306K\315\331\254\314\207 \n\022\013!m\2478;% \207\364\033r\324\0277\006\224\010\013\364\310\034\234_\307x`\302NZm\346\200\232\347j\257\233\035\222\346C\274\332\224\325<\312k\201Jh`\361S\2114R\256\221 \t\244\t-\2115#[9\357\335\320\310\226\2303D\204\223E\226\033:\313\034c\002\244\223\310\037\255n\217\220\265\242D\370(\315^\250\332@1\220\357=\241\037\301\007\322\347V\005\372(\274$\253o\\ \247\2750\206\316B\357L\3320F\326\346\r!\315\325}v\364/S\242Y\033\252\341i;R\226?\006\261$\225\203\367\005\355ZD`OP'\230c\324$W\220H/\031R\004WB d[&\215?\013\212\002\017\263\302\241\203\321""\264\014\036\206u\300\317\212Mf\006\320 \360\355\220C\000h\333\273!1\321\330\226\034\006\343\024\255^\234'\202#\323a\373X\260L'\312\220\303\"\232\002^J\243\0309\330\274\334%\367>\200+G\301\014\370\271\0137\026\311?\361\n\0173\370\253\r@\344\243\036|\242\301\270\002%?;@1\031\322y\2753\t\234\0145T\2640\245\265\331z\257\016-2\017\\z\324\014\313\026\010\326q\216\241\nG\226=-[7o\310\357\210\260\276\354@\002\020gc&\315c)\203\321\216\351\220\271\347 \231\226%Q\225#\335\331\345:(Jb\335\370^\035\0077T:\242\274mt\"\254`X\200x\256\246$\030w\352\203\342\270\243:\230\210\226pYb\235\025\200\360{\027NWE\330\310_\363\222Tu\2168\341bJ\371\350q\014\210N\263O\020^\231\211\261$\355\260\302\350\312\026\017^\267\337K\205\341\354\266\033bx\275u\355i\251F\302\0330\026\261\254*\"j\000|\356(HjX\014\330\333.\376\201\235l\014}O\005\235\361C\312\231xrK\227H\037\327\320\274D#\"M\016\257\303\233\3274\356]\226\027W\360P\277\334\004+\304$ \350\\9\tC[4\t\272\021\202Q\311C\217\217\302e-\321}\261\242\014V/\305\016ND\216)#d\302\333<\212F\001B\2311\366\216\331\276\270\340\342\201\001\256>\322\376\000DH\021\210\225kIS\224\242u\236\225rP\021\363\020,b\020ci\265\202B5\240\235\313\346\251{=\037\2510\024\210\206\2223\225\023\013\027\243\0014#\231\232\"\220\233\206\017\022\264l\251I\\\220\236\335\265a\222{*\301\256V|*+\000\240\315I\004\330\317\030D\2732\017>\025\365\304\203\200,\266\353\355\020\222\"W@|\"\016\370\326\267\016N\007\030\001\201\026\375\265\001\002\356\017\353\250\241ct\314# \023AU\244\220`\233u\205\254v\260\370,X\001\020N*\271\253r\3062\006\236.\372/\204&\250\334\342\300\266\034\342\017\034,\0236\343\374\274\323\250\034\237\232\371\352&\320\273\370H\257h\330F'\227\254t\227\364e0\007\367\332\233\233Zu\241\240\335`\\^\263\036C9\374\227\006\257\001\320\343_\211\251\330QPf(\245\344\310\317\014W|\214`\r\216\004\316\007%p\243\3104\223\252A\213\316\377\262\352\245\017C\300\007hKB>\270\014\245B\2028DSh\201\371g\241\254$\313\327'\324\251\335\235-l*X\002]\221_\226\021yu\231\344\357\273X\305\225\310\017""\362";
    PyObject *data = __Pyx_DecompressString(cstring, 2079, 3);
    if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error)
    const char* const bytes = __Pyx_PyBytes_AsString(data);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) }
    #endif
    #elif (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (2249 bytes) */
const char* const cstring = "BZh91AY&SY5\254r\247\000\000\204_\200D\020R\377\377\372\277\377\337~\277\377\377\361`\t(Y\017\257m\236\256\335\230\327\013m\267v\315lw;\251s\033\273\230J\021O!\250\304\332\nj\237\245=\221\252xQ\352mOP\375H\311\246zP=OI\372Pz\200\224&@ \246A2e?T\017S@\320\000\r\000\003\324\r\002*x\023Si1\0004\014\201\240\000\032\000\000\000\000\221\020BjzhI\215\024=OI\351\246\243@\03224\000\000h\3208i\246\010d4\323#&\020\r4\001\204\321\246L\000 h$\220&\2010\202f\205\032z@\310\000\000\000\000\000\363T\245\270\237\016\007\226oE\370\002\255\033uUU?\310\n8U\027.\226\360\376,t8\306\327.Y\006\374\351c\322\266t^\026\017\t\336\177\245\312-\274\311f?Zn\356\326\244\021\014\355\313B\342\224\356\303\316\302\335\221\366y\036\217oM\315L\320\257u|7\244\3447\355\260\362h\376\260k\225P\242c\021l\356:\0172\211V\272\361\2421\231\223\325\314,\256\275]\024x\256\232eAm\032L\321\355K\312\027\335\342y\324\rW\275\024U\3204\221,\013\347\025\201uo\310\357q\272\212t\2111\356\217\177\253\362\356=\347Nu\227.\322L\273\027\270\336\315\016\2534\2220\214\330*PDz\365\230\234\336\025\322\251\214\370\227V!C\345#\256VE\371B\330\206\363\263BdA\342y\312&<\2014A\274\243\201\271 \tI\235\337\324y\306\231\350e\000J\240S0\302\315\035\007^\017\224,ZaP\367\315\262\315V\214i\216\316\321|L\034k\231\334\313\356\344<\310\3609|\007\n~\207\341\022\275\335\177Eg_\211\361\014\340n\350\334x\252!\320uoG\006aIQ\233\215\371\325\225\227\2620\233\273\373i\251eE\370\262\267\313\001E\275\3718\2330\264Q\200_\200\3410\2741\254\260\034\020\373\343\304y\334\2445\362F\232\255\257\201\350\254\210\236@\207,\341\312KK\262\321?1\257\237\330,\257Y\226V\216\314j6\020\016\001h<\272\016\030\253q\3004\354\367\n\347=#\237\0315ai\233^U\340\031g9\037G\347g\230\\\202\275\323\230\225\335\305\2129\350\301\340\367\350\270\354\222\240s\226\357\303\2509aV$C\030\307\334\316\032\342v\344\034Bl\222\326\310\177\024\213\2570R-qs\266\2217\"f\202\006\242\343\320l?DQ'\361_Nm\207\367\224\334\003*\255\257.\251\360w:\256n\365t7\006\273]\215>N\226\205o\255\341\210Ev\355\265\017""\370\2527\255\216\020\327\221\340\376\336\352\353\255b\223'\306\377\034|\362\277Y\355GU\014\225\376\037\027Ee\347\214D\315\026\321\207\315k\215\275\333-\216\327\257\232\327\302\250\327V?\235\270q\302\304\354\214a\317\216\014(\242\2649g\370\202G\026\325\3357Z\316\374\350n\341R\265=;O\274\3255\3714\257\364l\014\355I\3373\0231\020i\315B\317f)D\003\014\3010\341V\252`\272*eQ(m\023\304\231\r\245\341Z\243\326VR\374\31662\363\026\213\3726\235I\370\341C\032\240\363\274k4,RJ\263B8\226#M1\035\321\r7\230\271\274\273:<\3575Q\207\224\023@\306<=\036\014~\230\222/P6j\310\254E`\374_\252\303\372\205\216\264\337\367>\242\232\331N\263j;\344F\023o\337\214\367\324'`^3\246\223\202\210L$^@\224d\211(\263\031V\225\231R\216t\222\025(El\003Q\264\214\320\307\032\242#\363\307\177\220j\000\215\224:\306#Pc\314{2\033\371\r\326\317\302lP\262\010\375\232\201\215\261\334\242\225\203[\360\333\276\201v\253\333ZR\\[\244\345=\350\276\246<%\223\006\270hn\025\312\217X\214\321\201(D8C\002\037\022::H\313o\246\205<{7\367_\273\246\247m\243E?iQD\203\005\322(\252\026\341(%T\2504\333\354\326\301\257\034\210\273]\324\306\020\272;5\337e\027\301\022gM)\003\210L\207\002l p\365\353\310\252)\300\2008f\262,\303\377\031\321\001\321t\r\224\347\350\276\376\243S\270.\274\322\321#;\264\013\236kyIJcfrz\305\224\353\264e\302eE\246&\223\005x\352\323\305\321\303F\322jt\216\217g\240\322i\270\322\2043\312\274\251\342\213\255\352D\0231\227J\005-;:D\004,:\326\256\353\023\223\310w\007lRN\222\223\255Y\310aFH\030NLPX\303\204\254IE\314\241\343\207\234\273\254(\266]TW\323\r\300\350<fZ\242\333\036\231\354\007\307\355\257K\370I\217h\250\313\224\207S\277\324\367z\305\274\2319\303e\366\033\212v\252\223\243j\274\362\330\353\270g\2325c\246X\325%\014\226\276\235\225{5p\3665\203\310u\242\013\374\003\331\207^\007/O'O*\323\177\2320D\372\231\r1\266\233md\373\004\277%\307%\034{\365\021\367Bg\016\272s\2244\216\314\0043(\343F+\273M.v\223\333\232a3\264Do\"\023E\325\017\203\177\\G\275\247\313\230Yhj\331\005\311e\245&@0a\204\354\206q\221\"\250\271r\300""\340\301\010\315\342\344\030\241\237\244\260\347\244\312\020\373\024\032#\256\326\241{\031b\216\213\274\271\266H\333p8\r\0256\361d\335\025\312]\275\365\213\003k\"51\267>\264\355\357\260\375\212\334\223\007\222\274\306\202\202\356\351\203Rhml\344\313X5\326\201c\366\325\213\303\233M\0070\374\211M\311\232\033Lq`\223\272*\312\302\237\310\370\357\245\001{\t\331`H \222\263\204\212:\006\360\220\313!*\330\345\321_\210Ds\261\271x\342\215e[&_\233v\264\331t[o]Yg\341,\367\031\314;y\241j\"\222]\321\2653\tmm;\030v\271J\354\324\312\322*\267\350\340\245M\371\356\254\253q\270R\351eD\262Tj&&:\331\021vZ\310\032\375D\230\326\254\231\231m\271\034\302q\022z(,\225\367B\306\330'\021f\303#\0312\271\005J\266\2535\3470\253\354\265\2064\261d\013\005\025\320\304\007\226(\263\270\\n\354M2&JP\217\031\321ro\004\006q\307u9\037\001\030\251\244@\210\2478\251\002(uv\204\246m\252\252\371jN\311\010\242\223\246\346Q\223r\312$f\360\224\031P\351\346\331\325\337l\256\312\324X)^\244\311\206H\233\231\353\030\026O\221\2220g\234\351\252\263h\251\370\213j\343\274\205\001\216v\035\277\026\264\036\224\335\216\226h\335\006D\225d\275O<\241R\335\020`V\301G\361\234{\303\262QP\334\250\224\306\202O\256\301E\275\303\\I\221\260cR\206\344\201x\034\023\010\350\225\350$\252\220,\311\216C\000q!L\355\270\244^\310Q\213i\223\212\213\300E\335\324X9\002<ZW\nM\217\177>*\274\2151ul\213\206M,\240\217\014\"\031-^\262O\202\263X\277\271\260/\272\366m\372\200\357\351\312\354\263PQ\213\303\267\372%%\247\t\372\313b,t\362l\325\025\016\265\345\207\205\271S\307<\035\220\251\302\216\231>0\375s\245<\373b\030\372\313\"\371\311k\010\321\024!\r\004E$\255\242w\016\226IJ\252i'fr=\030G\367TD\266\250E\3075\232\\\020\177\206h\304\275M\305\3673\202\325\036\260Th\340L`^\366;=\321\252\234[d\026+\303eJ\214FS\022\026\315\035\214\250\031\337\022\201yy\330\307\315\006\005\364\014b\032l\240\3660\326\263\212\345\275\245\345\215\345\234\000n\024\361\010\303\262\322\261X\347\272\226cs\232o\253\032\010^vT\002\207\252\031\263\035$\004i}\322\365\362\215'C&\021\231""\330R\277\325\207*\256\t\027'0\340[\244\021\007\343F7\306\245PX\206!\255\361)\021\032\346]\026%\262\330Z`(\244y\254\313\035\032\257D\270\320\212\312F\305\261es`\310r\210\272i\021\342&{\370F\034\243\271\003\320\277\024\240J\016\371\340HC\r\254\257\rs\334\212\315\231L\005\321\007TtbIZ\\\2042\"\343hiBw_\004\234u\340\237*\231%\310j\343\377\213\271\"\234(H\032\3269S\200";
    PyObject *data = __Pyx_DecompressString(cstring, 2249, 2);
    if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error)
    const char* const bytes = __Pyx_PyBytes_AsString(data);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) }
    #endif
    #elif (CYTHON_COMPRESS_STRINGS) != 0 /* compression: zlib (2077 bytes) */
const char* const cstring = "x\332\235Wmo\033\271\021\316\347\373\025D\200\342\362\342Ul'~\271+\214B\221\225F\255\022\t\262}p\323+\010\212KI\214W\344\036\311\225\265\367\241\277\275\317\220\273\362Zq\256E\ry\227\034\016\311y}f\366e\357\350\360\350\020\277\267G\307G\307\207xdoON\016O\216O\336\236\236\036\236\036\235\036\237\235\035\236\035\237\235\234\237\037\236\237\236\237\375\364\323a\366\336\327\353\271-\264\034\334L\377\301\0077\227}\376\313pv5\232|\276h(\037>\\\363\253\353\376\365hpq\355*\025\251\037G\323\035\333av\371\261\177\305\307\223\376%\037\364\307\343\367\375\301\337\023\351\352z2\033>\320\206\037\204.T\316\202e\013mrF\227\261\225\022\271r\276\227\215\3766\272fR\024\305\\\310;\346\253\262\264.0\247~\253\264S\236\311\nr\260\205\263\353\264\357\350\270w\376:\033\177\266F}\266A\261\260\022\201\r\352\260\262\206i\317rU\350\271r\"\250\242f>8-\203r\304d\330t8\315\336\235\277c\002\"8\365U\311\340q\335\\\026\302{\\d\027l^\351\"h\303B]*\337c\243\005\253m\305\214J\262\227\340\353n\010+e\230W\201\006\354Ga\214\r\"hk8\266k\263\374\221\345P@\006\275Q\264\373\203(\274\352e\223\343\037\256\002\024w?\263\033\017\256\270\271PK!\353\007#@\017i\215\327\340\303\325\271*q\0164\312\017XY(\341\025\363\367:\310\025\035[a&\347|\243\334\305\363\257:<g\332\370\000\323f\267\322\256K\230\335e\267\2056w\312\365X\257\367\363\017\177\351\t\301\006\257_\263v\231.k\214\235\303\000\201A\r\266\260\225\351\334\326\350\250\314F;k\326\312\004\266\021N\213y\241\330\340\366V\3449\307.\225\t'W\027~\315\347\332\2741\233\322UFe\022\322\025V\200C\254!j4\023\233+\310\271\200\025\272:\225J\352E\315t`j[\"0u(jl\366\301:\365\177\357\206a\330\272\362q\323\363d\347\347\314:\026\215E\302is m\256H\354,\333\331\014dx\217K\\\375F\033YT\271\302jYT\276'\3132\305d\353.\317\204S\314\032\004\234\330 \322\243Y\020\215cm\252\355>kcjv\372\216\315!\353\344JV\213E\350\225\333\274\031\324\333\177\277\351\311\252\254\337\264\233\270\024r\245\210\304#\317m\300\372\243\351\352a\326[\345\321\334\203f\3634\270_\375\253\013\374\277\370g?\373\"\262\337""\371\277~\275\177\365\022\204?\347\311\266\377\013k\226K\236\253M\317\346\332\223~\312\3203[LG\203\245\314\262\24521\347\262h\312\030\006d\312*(~\300\226p\031[\t\017\247\301\236\224\0221\301\330\0138\n\023\204\335J\227d\245\334\"\253\310\301-\010\354\314\246}\2720\317\212\254\210zBt$\233\244\031\022\336z<N\337\275\301\263\273\332\023d\212GyE\307/\221\222&+\312\260r\310\2246\275\333\010\372\306~\220\2335A\374-\357\276\005\033\346H\336\347V\032Y\344\332\263(\n[\316\003\232\314mX\231\215\224\337\317\310&\247\330\"\341)@`\305\020\267-\271Q/\262f\266gcH6\000\021S0\027AD\370#\266\226(\252-\027\316u\231S\312\355s'j\303N`H\360\005\\59\346\242&M\367\316\354\252\270\243E\220\376\003D\2130\013\306^\274\277sh\234wO\214\204\006\357\016\"B\275\332\335\"\334\262\"\240\362\260RQP\366;\265\266\033X\r\016\022lQ\205\312\021-\336N\351G\231\367'\317\276\363\313\374\nI\236g>\344\027\362\365\353\243\263\354a\030\205y:\310\356\251D\220_wN'{\266\nD\334\210|Q\372z\007T(E\244\215'\314\243TI\265xp<h\007\263f\360\236\217Q{'\237\246\343\341\355\023$~9\271y?\036vWf\303\376x\177\276\317vu\375\315\231\017\244'\230\037\235\331\314\037\261]\036\177I\203Y\253\302\227\343\313v\360%\265\033\375\301\307!\277\034\315\342\354\362\346\3234\365$W\223\233\331`\310'\237\371p6\233\314Pmx\233m\237\204\021K\345h\252\362\251\263Ry?D \273AB\361\341V\252\222\n\362\360\026\215\014n\033\335r\364\032{\333\3718\026\205=\352t4\035r>\255\267\370\277D\017\301?\253m\230\251\305\265Z\227\005\002\227\336\326\t\207E\0024\353j~\035\321dl\021\010\215\263\367\262\241\033\307\335\242\330\206\303~\246<\212\363G\2050\3255*\nw\334)D\262!\334\225\210eS\225\022\345^q\2156\304\t\251H\247\246\252\361\366M\270\221\212\njF.\244-\353\005\350\022\362\207\346\311\241\245\241E\036Yx\314p\034\212fj\201S\023\265W\212\260\242\216\016\262`\352B\304\336.\000s*L\364\337\343\235\346!U\253]u\243\345(G;\350\265e\270CI'cJ5\356as\334\315\227*\360\006\205xC\"\373\334\3472\366\205\337 \372\267\260\r\353\301\202\250p\311-\273\363\3217\0211\231\276K\345<""\247\260\300\323\341GN\311\253u\251\242#8\224\r\304\242\0140\210\352\"\224o\364W[I\375\241\242\367F\024\025\006J\362\265\315+\024\325\255\366\201[h\205Q\240g\t\270\000&;\262(\355\342\234\034EO(\205\003s-\250\331B\034\362*\350B\207\272\273\222Zl\236\267!\n3\341\367\013\272n\352R\311hsk\013\362\014o\3339\"\246\310\300\2568\261f\241\227\264\036g\024\013\344\365hq\324\252\335\204\336\234\316$\307C\302\245\263U\211\256;/\024J\377\n\205y\245\266\271^*\0374\245N\000\245i\254\264\307]\332\223f:\250\265Gg\366\325j\223\2325\260\025\324G\221[\224+(\2738_\013\2047\236w\n;\375Zl\275\376]\255\363\023\336\030r\367\246\317\006N\3457\315=\371\211\363\364\304[\335\363\250ES@\315\006!\314m\304\014o\341L\353I-J\005\372G\370/,e\377\302:\250P\242\206! \254\214\271\3029\352\007\217)\351\253u\2325\246\240ar\036\215*SjyG2\356\003\321\336\362\023P\265\307\361$r\3553u`io\311G\207Rh\355\372\254\304\261\t\024\nd\241\337*Q$k9E\307\320;\257d\\kF)\305:\004\265\245!\355\202M\275\242f\0207\365\253`\373d\255\370\211\004B+6\206\035M\243L\355\370I\001\211\230D\302(W\013Q\025!\216\033kw\306\255h~E\311\301\375\232\"!\305DL\243\2354\370^\240-.P\027\262\362\361\373\014O[\005\372\2024K|\364\225\251\276`\204\000\016hl}\355\361K\351\201\250-Sb\202\016iRc\211\215\241\000r\360\"X\355vh\\\225\300s\205\254\316\021G\036}\275C\332\222\200\021\016\240\341\246IP~\177/\234\241\177\022\341\036ljZO\346\364\341\312^\275\000\026\263&\314\356t\211t\245X\225\253\227\317h\341\373\253)\247Sk.E)\346\0215\"\031@\247\341?@\207\t\317\232?\256=Gw\336E\357g\211\257\037`\032t\247j\220N\033\354\016\373$\276Z\367\337\271\264\001\227J\365\032p\271\r#\177\t\3539[\253<\321G\006&\321\371Ly[9\251>F4y\264\364\013\231,Q>\241\275s\365C\210%\352T)\327\227\344\271~A>\251\207\351;\342\217\215\204\300\005J\251\277\356\300\022bU\262e\266\363\257\374],2'\361\003\021\023z\356*\320\234\372\331W/\036|u\300\276{\023\341+\374\035\350P\3606\245\371\211\212\373\037\220\265\270\251";
    PyObject *data = __Pyx_DecompressString(cstring, 2077, 1);
    if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error)
    const char* const bytes = __Pyx_PyBytes_AsString(data);
    #if !CYTHON_ASSUME_SAFE_MACROS
    if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) }
    #endif
    #else /* compression: none (4579 bytes) */
const char* const bytes = ").1010010312120121-35505253660616277072758808687990-BsymbolicCUPY_CUDA_VERSION=CUPY_CUFFT_STATIC=TrueCUPY_HIP_VERSION=0-DHAS_LOAD_CALLBACK-DHAS_STORE_CALLBACK-EFailed to find CUDA headers.-IJIT callback support requires cuFFT from CUDA 12.8+-LNoneNote that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False.-O2\nStderr: Using the legacy callback is considered deprecated, please switch to use cb_ver=\"jit\" instead-Xcompiler-Xlinker. ..:\n?.aa C++ compiler is required but not found, please set the environment variable CXXadd_note-arch=sm_bin/nvprune-ccb_load_name cannot be inferred, please specify it explicitlycb_store_name cannot be inferred, please specify it explicitlycb_ver must be \"legacy\" or \"jit\"-ccbin,code=sm_--compiler-bindir_core/include--cplus.cppcuFFT callbacks are only available on LinuxcuFFT callbacks require 64 bit OScufft.pxdcufft.pyx~/.cupy/callback_cachecupy_cufftXt.cucupy_cufftXt.hcupy_cufft.hd_loadCallbackPtr\\s*=\\s*([A-Za-z_]\\w*)\\s*;d_storeCallbackPtr\\s*=\\s*([A-Za-z_]\\w*)\\s*;-dc_dev.odisableenable-fPICgc--generate-code=arch=compute_, got  has incorrect type (expected hipFFT does not support callbacksisenabled-l-lcufft_static-lculiboslib64/libcufft_static.aload callback is not given-lpthreadneed to specify d_loadCallbackPtr in cb_loadneed to specify d_storeCallbackPtr in cb_storeneed to specify either cb_load or cb_store, or bothnvcc is required but not foundnvprune failed with sm_nvprune is not found-o.oonly use cb_load_data and not cb_load_aux_arronly use cb_store_data and not cb_store_aux_arrpassing an ndarray to cb_load_aux_arr or cb_store_aux_arr is deprecated, please switch to pass arr.data to cb_load_data or cb_store_data instead, the *_aux_arr arguments will be removed in a future release.pxd.pyx%s %s %s %s %s %s %s %s %s-shared-std=c++17--std=c++17store callback is not givenwhen both cb_load and cb_store are give""n, they must be of the same typeCUFFT_C2CCUFFT_C2RCUFFT_CB_LD_COMPLEXCUFFT_CB_LD_COMPLEX_DOUBLECUFFT_CB_LD_REALCUFFT_CB_LD_REAL_DOUBLECUFFT_CB_ST_COMPLEXCUFFT_CB_ST_COMPLEX_DOUBLECUFFT_CB_ST_REALCUFFT_CB_ST_REAL_DOUBLECUFFT_D2ZCUFFT_R2CCUFFT_Z2DCUFFT_Z2ZCUPY_CACHE_DIRCUPY_DUMP_CUDA_SOURCE_ON_ERRORCXX_CallbackManagerCalledProcessErrorCompileExceptionEXT_SUFFIX_JITCallbackManager_LegacyCallbackManagerPIPE__Pyx_PyDict_NextRefTemplateTemporaryDirectory_ThreadLocalcb_loadcb_load_aux_arrcb_load_datacb_load_namecb_storecb_store_aux_arrcb_store_datacb_store_namecb_vercheck_returncodecleanupcline_in_tracebackcompile_compile_with_cache_cudacopyfilecreatecreate_plancuda__cuda_array_interface__cuda.pathfindercudartcufftcufft_static_cupycupy._environmentcupy_callback_cupy.cudacupy.cuda.compilercupy.cuda.cufftcupy.fft._callback__cupy_get_ndarray___cupy_vercwdcythond_loadCallbackPtrd_storeCallbackPtrdecodedev_load_callback_kerdev_store_callback_ker__dict__dirdirnamedumpencode__enter__enumerateenvenvironexc_typeexc_valueexec_moduleexist_ok__exit__expanduserfft_type__file__find_nvidia_binary_utilityfind_nvidia_header_directorygetgetVersion_get_bool_env_variableget_cache_dirget_config_varget_cuda_pathget_nvcc_pathget_path__getstate__grouphandlehashlibhexdigestimportlibincludeisdirisfileitemsjitjoinlegacyliblinuxloaderlocal__main__makedirsmaxsizemd5__module__module_from_specmodulesname__name____new__nvccnvprunenvrtc_optionsopenospathplanplan_infoplatformpopprealloc_plan__pyx_checksum__pyx_state__pyx_type__pyx_unpickle__CallbackManager__pyx_unpickle__JITCallbackManager__pyx_unpickle__LegacyCallbackManager__pyx_unpickle__ThreadLocal__pyx_unpickle_set_cufft_callbacks__pyx_vtable____qualname__reread__reduce____reduce_cython____reduce_ex__renamerunsearchsetAutoAllocationsetCallbacksetJITCallbackset_callbacksset_cufft_callbacks__set_name__setdefault__setstate____setstate_cython__shutil_smspec_from_file_locationsplitstartswithstderrstdoutstringsubprocesssubstitutesyssysconfigtempfile__test__""threadingtlsto_ltoirtracebackupdateusedforsecurityutilvalues__version__wwarnwarningswritePyObject *(int __pyx_skip_dispatch)\000int (int __pyx_skip_dispatch)\000get_compute_capability\000get_device_idint\000\000\000\000\000\000\000\000_is_hip_environment\000deviceAttributeComputeCapabilityMajor\000deviceAttributeComputeCapabilityMinor\000errorContextIsDestroyed\000errorInvalidResourceHandle\000errorInvalidValue\000errorMemoryAllocation\000errorPeerAccessAlreadyEnabledint (int __pyx_skip_dispatch)\000runtimeGetVersionstruct __pyx_obj_4cupy_5_core_4core__ndarray_base *(PyObject *, int __pyx_skip_dispatch)\000_convert_object_with_cuda_array_interface";
    PyObject *data = NULL;
    CYTHON_UNUSED_VAR(__Pyx_DecompressString);
    #endif
    PyObject **stringtab = __pyx_mstate->__pyx_string_tab;
    Py_ssize_t pos = 0;
    for (int i = 0; i < 297; i++) {
      Py_ssize_t bytes_length = index[i].length;
      PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL);
      if (likely(string) && i >= 111) PyUnicode_InternInPlace(&string);
      if (unlikely(!string)) {
        Py_XDECREF(data);
        __PYX_ERR(0, 1, __pyx_L1_error)
      }
      stringtab[i] = string;
      pos += bytes_length;
    }
    for (int i = 297; i < 301; i++) {
      Py_ssize_t bytes_length = index[i].length;
      PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length);
      stringtab[i] = string;
      pos += bytes_length;
      if (unlikely(!string)) {
        Py_XDECREF(data);
        __PYX_ERR(0, 1, __pyx_L1_error)
      }
    }
    Py_XDECREF(data);
    for (Py_ssize_t i = 0; i < 301; i++) {
      if (unlikely(PyObject_Hash(stringtab[i]) == -1)) {
        __PYX_ERR(0, 1, __pyx_L1_error)
      }
    }
    #if CYTHON_IMMORTAL_CONSTANTS
    {
      PyObject **table = stringtab + 297;
      for (Py_ssize_t i=0; i<4; ++i) {
        #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
        #if PY_VERSION_HEX < 0x030E0000
        if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1)
        #else
        if (PyUnstable_Object_IsUniquelyReferenced(table[i]))
        #endif
        {
          Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL);
        }
        #else
        Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT);
        #endif
      }
    }
    #endif
  }
  {
    PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 0;
    int8_t const cint_constants_1[] = {0,-1,1};
    int16_t const cint_constants_2[] = {11303};
    int32_t const cint_constants_4[] = {52696597L,146004246L,238673578L,258762224L,265177976L};
    int64_t const cint_constants_8[] = {4294967296LL};
    for (int i = 0; i < 10; i++) {
      numbertab[i] = PyLong_FromLongLong((i < 3 ? cint_constants_1[i - 0] : (i < 4 ? cint_constants_2[i - 3] : (i < 9 ? cint_constants_4[i - 4] : cint_constants_8[i - 9]))));
      if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error)
    }
  }
  #if CYTHON_IMMORTAL_CONSTANTS
  {
    PyObject **table = __pyx_mstate->__pyx_number_tab;
    for (Py_ssize_t i=0; i<10; ++i) {
      #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
      #if PY_VERSION_HEX < 0x030E0000
      if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1)
      #else
      if (PyUnstable_Object_IsUniquelyReferenced(table[i]))
      #endif
      {
        Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL);
      }
      #else
      Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT);
      #endif
    }
  }
  #endif
  return 0;
  __pyx_L1_error:;
  return -1;
}
/* #### Code section: init_codeobjects ### */

static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) {
  CYTHON_UNUSED_VAR(__pyx_mstate);
  return 0;
}
/* #### Code section: init_globals ### */

static int __Pyx_InitGlobals(void) {
  /* PythonCompatibility.init */
  if (likely(__Pyx_init_co_variables() == 0)); else
  
  if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error)

  /* AssertionsEnabled.init */
  if (likely(__Pyx_init_assertions_enabled() == 0)); else
  
  if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error)

  return 0;
  __pyx_L1_error:;
  return -1;
}
/* #### Code section: cleanup_globals ### */
/* #### Code section: cleanup_module ### */
/* #### Code section: main_method ### */
/* #### Code section: utility_code_pragmas ### */
#ifdef _MSC_VER
#pragma warning( push )
/* Warning 4127: conditional expression is constant
 * Cython uses constant conditional expressions to allow in inline functions to be optimized at
 * compile-time, so this warning is not useful
 */
#pragma warning( disable : 4127 )
#endif



/* #### Code section: utility_code_def ### */

/* --- Runtime support code --- */
/* Refnanny */
#if CYTHON_REFNANNY
static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) {
    PyObject *m = NULL, *p = NULL;
    void *r = NULL;
    m = PyImport_ImportModule(modname);
    if (!m) goto end;
    p = PyObject_GetAttrString(m, "RefNannyAPI");
    if (!p) goto end;
    r = PyLong_AsVoidPtr(p);
end:
    Py_XDECREF(p);
    Py_XDECREF(m);
    return (__Pyx_RefNannyAPIStruct *)r;
}
#endif

/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */
#if CYTHON_FAST_THREAD_STATE
static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) {
    Py_ssize_t i, n;
    n = PyTuple_GET_SIZE(tuple);
    for (i=0; i<n; i++) {
        if (exc_type == PyTuple_GET_ITEM(tuple, i)) return 1;
    }
    for (i=0; i<n; i++) {
        if (__Pyx_PyErr_GivenExceptionMatches(exc_type, PyTuple_GET_ITEM(tuple, i))) return 1;
    }
    return 0;
}
static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) {
    int result;
    PyObject *exc_type;
#if PY_VERSION_HEX >= 0x030C00A6
    PyObject *current_exception = tstate->current_exception;
    if (unlikely(!current_exception)) return 0;
    exc_type = (PyObject*) Py_TYPE(current_exception);
    if (exc_type == err) return 1;
#else
    exc_type = tstate->curexc_type;
    if (exc_type == err) return 1;
    if (unlikely(!exc_type)) return 0;
#endif
    #if CYTHON_AVOID_BORROWED_REFS
    Py_INCREF(exc_type);
    #endif
    if (unlikely(PyTuple_Check(err))) {
        result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err);
    } else {
        result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err);
    }
    #if CYTHON_AVOID_BORROWED_REFS
    Py_DECREF(exc_type);
    #endif
    return result;
}
#endif

/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */
#if CYTHON_FAST_THREAD_STATE
static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) {
#if PY_VERSION_HEX >= 0x030C00A6
    PyObject *tmp_value;
    assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value)));
    if (value) {
        #if CYTHON_COMPILING_IN_CPYTHON
        if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb))
        #endif
            PyException_SetTraceback(value, tb);
    }
    tmp_value = tstate->current_exception;
    tstate->current_exception = value;
    Py_XDECREF(tmp_value);
    Py_XDECREF(type);
    Py_XDECREF(tb);
#else
    PyObject *tmp_type, *tmp_value, *tmp_tb;
    tmp_type = tstate->curexc_type;
    tmp_value = tstate->curexc_value;
    tmp_tb = tstate->curexc_traceback;
    tstate->curexc_type = type;
    tstate->curexc_value = value;
    tstate->curexc_traceback = tb;
    Py_XDECREF(tmp_type);
    Py_XDECREF(tmp_value);
    Py_XDECREF(tmp_tb);
#endif
}
static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
#if PY_VERSION_HEX >= 0x030C00A6
    PyObject* exc_value;
    exc_value = tstate->current_exception;
    tstate->current_exception = 0;
    *value = exc_value;
    *type = NULL;
    *tb = NULL;
    if (exc_value) {
        *type = (PyObject*) Py_TYPE(exc_value);
        Py_INCREF(*type);
        #if CYTHON_COMPILING_IN_CPYTHON
        *tb = ((PyBaseExceptionObject*) exc_value)->traceback;
        Py_XINCREF(*tb);
        #else
        *tb = PyException_GetTraceback(exc_value);
        #endif
    }
#else
    *type = tstate->curexc_type;
    *value = tstate->curexc_value;
    *tb = tstate->curexc_traceback;
    tstate->curexc_type = 0;
    tstate->curexc_value = 0;
    tstate->curexc_traceback = 0;
#endif
}
#endif

/* PyObjectGetAttrStr (used by PyObjectGetAttrStrNoError) */
#if CYTHON_USE_TYPE_SLOTS
static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) {
    PyTypeObject* tp = Py_TYPE(obj);
    if (likely(tp->tp_getattro))
        return tp->tp_getattro(obj, attr_name);
    return PyObject_GetAttr(obj, attr_name);
}
#endif

/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */
#if __PYX_LIMITED_VERSION_HEX < 0x030d0000
static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) {
    __Pyx_PyThreadState_declare
    __Pyx_PyThreadState_assign
    if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError)))
        __Pyx_PyErr_Clear();
}
#endif
static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) {
    PyObject *result;
#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
    (void) PyObject_GetOptionalAttr(obj, attr_name, &result);
    return result;
#else
#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS
    PyTypeObject* tp = Py_TYPE(obj);
    if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) {
        return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1);
    }
#endif
    result = __Pyx_PyObject_GetAttrStr(obj, attr_name);
    if (unlikely(!result)) {
        __Pyx_PyObject_GetAttrStr_ClearAttributeError();
    }
    return result;
#endif
}

/* GetBuiltinName */
static PyObject *__Pyx_GetBuiltinName(PyObject *name) {
    PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name);
    if (unlikely(!result) && !PyErr_Occurred()) {
        PyErr_Format(PyExc_NameError,
            "name '%U' is not defined", name);
    }
    return result;
}

/* GetItemInt */
static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) {
    PyObject *r;
    if (unlikely(!j)) return NULL;
    r = PyObject_GetItem(o, j);
    Py_DECREF(j);
    return r;
}
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i,
                                                              int wraparound, int boundscheck, int unsafe_shared) {
    CYTHON_MAYBE_UNUSED_VAR(unsafe_shared);
#if CYTHON_ASSUME_SAFE_SIZE
    Py_ssize_t wrapped_i = i;
    if (wraparound & unlikely(i < 0)) {
        wrapped_i += PyList_GET_SIZE(o);
    }
    if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) {
        return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared);
    } else
    if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) {
        return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i));
    }
    return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i));
#else
    (void)wraparound;
    (void)boundscheck;
    return PySequence_GetItem(o, i);
#endif
}
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i,
                                                              int wraparound, int boundscheck, int unsafe_shared) {
    CYTHON_MAYBE_UNUSED_VAR(unsafe_shared);
#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
    Py_ssize_t wrapped_i = i;
    if (wraparound & unlikely(i < 0)) {
        wrapped_i += PyTuple_GET_SIZE(o);
    }
    if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) {
        return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i));
    }
    return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i));
#else
    (void)wraparound;
    (void)boundscheck;
    return PySequence_GetItem(o, i);
#endif
}
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list,
                                                     int wraparound, int boundscheck, int unsafe_shared) {
    CYTHON_MAYBE_UNUSED_VAR(unsafe_shared);
#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE
    if (is_list || PyList_CheckExact(o)) {
        Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o);
        if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) {
            return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared);
        } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) {
            return __Pyx_NewRef(PyList_GET_ITEM(o, n));
        }
    } else
    #if !CYTHON_AVOID_BORROWED_REFS
    if (PyTuple_CheckExact(o)) {
        Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o);
        if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) {
            return __Pyx_NewRef(PyTuple_GET_ITEM(o, n));
        }
    } else
    #endif
#endif
#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY
    {
        PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping;
        PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence;
        if (!is_list && mm && mm->mp_subscript) {
            PyObject *r, *key = PyLong_FromSsize_t(i);
            if (unlikely(!key)) return NULL;
            r = mm->mp_subscript(o, key);
            Py_DECREF(key);
            return r;
        }
        if (is_list || likely(sm && sm->sq_item)) {
            if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) {
                Py_ssize_t l = sm->sq_length(o);
                if (likely(l >= 0)) {
                    i += l;
                } else {
                    if (!PyErr_ExceptionMatches(PyExc_OverflowError))
                        return NULL;
                    PyErr_Clear();
                }
            }
            return sm->sq_item(o, i);
        }
    }
#else
    if (is_list || !PyMapping_Check(o)) {
        return PySequence_GetItem(o, i);
    }
#endif
    (void)wraparound;
    (void)boundscheck;
    return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i));
}

/* HasAttr */
#if __PYX_LIMITED_VERSION_HEX < 0x030d0000
static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) {
    PyObject *r;
    if (unlikely(!PyUnicode_Check(n))) {
        PyErr_SetString(PyExc_TypeError,
                        "hasattr(): attribute name must be string");
        return -1;
    }
    r = __Pyx_PyObject_GetAttrStrNoError(o, n);
    if (!r) {
        return (unlikely(PyErr_Occurred())) ? -1 : 0;
    } else {
        Py_DECREF(r);
        return 1;
    }
}
#endif

/* PyObjectCall (used by PyObjectFastCall) */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) {
    PyObject *result;
    ternaryfunc call = Py_TYPE(func)->tp_call;
    if (unlikely(!call))
        return PyObject_Call(func, arg, kw);
    if (unlikely(Py_EnterRecursiveCall(" while calling a Python object")))
        return NULL;
    result = (*call)(func, arg, kw);
    Py_LeaveRecursiveCall();
    if (unlikely(!result) && unlikely(!PyErr_Occurred())) {
        PyErr_SetString(
            PyExc_SystemError,
            "NULL result without error in PyObject_Call");
    }
    return result;
}
#endif

/* PyObjectCallMethO (used by PyObjectFastCall) */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) {
    PyObject *self, *result;
    PyCFunction cfunc;
    cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func);
    self = __Pyx_CyOrPyCFunction_GET_SELF(func);
    if (unlikely(Py_EnterRecursiveCall(" while calling a Python object")))
        return NULL;
    result = cfunc(self, arg);
    Py_LeaveRecursiveCall();
    if (unlikely(!result) && unlikely(!PyErr_Occurred())) {
        PyErr_SetString(
            PyExc_SystemError,
            "NULL result without error in PyObject_Call");
    }
    return result;
}
#endif

/* PyObjectFastCall (used by PyObjectFastCallMethod) */
#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API
static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) {
    PyObject *argstuple;
    PyObject *result = 0;
    size_t i;
    argstuple = PyTuple_New((Py_ssize_t)nargs);
    if (unlikely(!argstuple)) return NULL;
    for (i = 0; i < nargs; i++) {
        Py_INCREF(args[i]);
        if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad;
    }
    result = __Pyx_PyObject_Call(func, argstuple, kwargs);
  bad:
    Py_DECREF(argstuple);
    return result;
}
#endif
#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API
  #if PY_VERSION_HEX < 0x03090000
    #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable)
  #elif CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) {
    PyTypeObject *tp = Py_TYPE(callable);
    #if defined(__Pyx_CyFunction_USED)
    if (__Pyx_CyFunction_CheckExact(callable)) {
        return __Pyx_CyFunction_func_vectorcall(callable);
    }
    #endif
    if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) {
        return NULL;
    }
    assert(PyCallable_Check(callable));
    Py_ssize_t offset = tp->tp_vectorcall_offset;
    assert(offset > 0);
    vectorcallfunc ptr;
    memcpy(&ptr, (char *) callable + offset, sizeof(ptr));
    return ptr;
}
  #else
    #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable)
  #endif
#endif
static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) {
    Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs);
#if CYTHON_COMPILING_IN_CPYTHON
    if (nargs == 0 && kwargs == NULL) {
        if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS))
            return __Pyx_PyObject_CallMethO(func, NULL);
    }
    else if (nargs == 1 && kwargs == NULL) {
        if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O))
            return __Pyx_PyObject_CallMethO(func, args[0]);
    }
#endif
    if (kwargs == NULL) {
        #if CYTHON_VECTORCALL
          #if CYTHON_COMPILING_IN_LIMITED_API
            return PyObject_Vectorcall(func, args, _nargs, NULL);
          #else
            vectorcallfunc f = __Pyx_PyVectorcall_Function(func);
            if (f) {
                return f(func, args, _nargs, NULL);
            }
          #endif
        #endif
    }
    if (nargs == 0) {
        return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs);
    }
    #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API
    return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs);
    #else
    return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs);
    #endif
}

/* PyObjectFastCallMethod */
#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000
static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf) {
    PyObject *result;
    PyObject *attr = PyObject_GetAttr(args[0], name);
    if (unlikely(!attr))
        return NULL;
    result = __Pyx_PyObject_FastCall(attr, args+1, nargsf - 1);
    Py_DECREF(attr);
    return result;
}
#endif

/* ExtTypeTest */
static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
    __Pyx_TypeName obj_type_name;
    __Pyx_TypeName type_name;
    if (unlikely(!type)) {
        PyErr_SetString(PyExc_SystemError, "Missing type object");
        return 0;
    }
    if (likely(__Pyx_TypeCheck(obj, type)))
        return 1;
    obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj));
    type_name = __Pyx_PyType_GetFullyQualifiedName(type);
    PyErr_Format(PyExc_TypeError,
                 "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME,
                 obj_type_name, type_name);
    __Pyx_DECREF_TypeName(obj_type_name);
    __Pyx_DECREF_TypeName(type_name);
    return 0;
}

/* JoinPyUnicode */
static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength,
                                      Py_UCS4 max_char) {
#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
    PyObject *result_uval;
    int result_ukind, kind_shift;
    Py_ssize_t i, char_pos;
    void *result_udata;
    if (max_char > 1114111) max_char = 1114111;
    result_uval = PyUnicode_New(result_ulength, max_char);
    if (unlikely(!result_uval)) return NULL;
    result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND;
    kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1;
    result_udata = PyUnicode_DATA(result_uval);
    assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0);
    if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0))
        goto overflow;
    char_pos = 0;
    for (i=0; i < value_count; i++) {
        int ukind;
        Py_ssize_t ulength;
        void *udata;
        PyObject *uval = values[i];
        #if !CYTHON_COMPILING_IN_LIMITED_API
        if (__Pyx_PyUnicode_READY(uval) == (-1))
            goto bad;
        #endif
        ulength = __Pyx_PyUnicode_GET_LENGTH(uval);
        #if !CYTHON_ASSUME_SAFE_SIZE
        if (unlikely(ulength < 0)) goto bad;
        #endif
        if (unlikely(!ulength))
            continue;
        if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos))
            goto overflow;
        ukind = __Pyx_PyUnicode_KIND(uval);
        udata = __Pyx_PyUnicode_DATA(uval);
        if (ukind == result_ukind) {
            memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift));
        } else {
            #if PY_VERSION_HEX >= 0x030d0000
            if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad;
            #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters)
            _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength);
            #else
            Py_ssize_t j;
            for (j=0; j < ulength; j++) {
                Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j);
                __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar);
            }
            #endif
        }
        char_pos += ulength;
    }
    return result_uval;
overflow:
    PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string");
bad:
    Py_DECREF(result_uval);
    return NULL;
#else
    Py_ssize_t i;
    PyObject *result = NULL;
    PyObject *value_tuple = PyTuple_New(value_count);
    if (unlikely(!value_tuple)) return NULL;
    CYTHON_UNUSED_VAR(max_char);
    CYTHON_UNUSED_VAR(result_ulength);
    for (i=0; i<value_count; i++) {
        Py_INCREF(values[i]);
        if (__Pyx_PyTuple_SET_ITEM(value_tuple, i, values[i]) != (0)) goto bad;
    }
    result = PyUnicode_Join(__pyx_mstate_global->__pyx_empty_unicode, value_tuple);
bad:
    Py_DECREF(value_tuple);
    return result;
#endif
}

/* RaiseException */
static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) {
    PyObject* owned_instance = NULL;
    if (tb == Py_None) {
        tb = 0;
    } else if (tb && !PyTraceBack_Check(tb)) {
        PyErr_SetString(PyExc_TypeError,
            "raise: arg 3 must be a traceback or None");
        goto bad;
    }
    if (value == Py_None)
        value = 0;
    if (PyExceptionInstance_Check(type)) {
        if (value) {
            PyErr_SetString(PyExc_TypeError,
                "instance exception may not have a separate value");
            goto bad;
        }
        value = type;
        type = (PyObject*) Py_TYPE(value);
    } else if (PyExceptionClass_Check(type)) {
        PyObject *instance_class = NULL;
        if (value && PyExceptionInstance_Check(value)) {
            instance_class = (PyObject*) Py_TYPE(value);
            if (instance_class != type) {
                int is_subclass = PyObject_IsSubclass(instance_class, type);
                if (!is_subclass) {
                    instance_class = NULL;
                } else if (unlikely(is_subclass == -1)) {
                    goto bad;
                } else {
                    type = instance_class;
                }
            }
        }
        if (!instance_class) {
            PyObject *args;
            if (!value)
                args = PyTuple_New(0);
            else if (PyTuple_Check(value)) {
                Py_INCREF(value);
                args = value;
            } else
                args = PyTuple_Pack(1, value);
            if (!args)
                goto bad;
            owned_instance = PyObject_Call(type, args, NULL);
            Py_DECREF(args);
            if (!owned_instance)
                goto bad;
            value = owned_instance;
            if (!PyExceptionInstance_Check(value)) {
                PyErr_Format(PyExc_TypeError,
                             "calling %R should have returned an instance of "
                             "BaseException, not %R",
                             type, Py_TYPE(value));
                goto bad;
            }
        }
    } else {
        PyErr_SetString(PyExc_TypeError,
            "raise: exception class must be a subclass of BaseException");
        goto bad;
    }
    if (cause) {
        PyObject *fixed_cause;
        if (cause == Py_None) {
            fixed_cause = NULL;
        } else if (PyExceptionClass_Check(cause)) {
            fixed_cause = PyObject_CallObject(cause, NULL);
            if (fixed_cause == NULL)
                goto bad;
        } else if (PyExceptionInstance_Check(cause)) {
            fixed_cause = cause;
            Py_INCREF(fixed_cause);
        } else {
            PyErr_SetString(PyExc_TypeError,
                            "exception causes must derive from "
                            "BaseException");
            goto bad;
        }
        PyException_SetCause(value, fixed_cause);
    }
    PyErr_SetObject(type, value);
    if (tb) {
#if PY_VERSION_HEX >= 0x030C00A6
        PyException_SetTraceback(value, tb);
#elif CYTHON_FAST_THREAD_STATE
        PyThreadState *tstate = __Pyx_PyThreadState_Current;
        PyObject* tmp_tb = tstate->curexc_traceback;
        if (tb != tmp_tb) {
            Py_INCREF(tb);
            tstate->curexc_traceback = tb;
            Py_XDECREF(tmp_tb);
        }
#else
        PyObject *tmp_type, *tmp_value, *tmp_tb;
        PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb);
        Py_INCREF(tb);
        PyErr_Restore(tmp_type, tmp_value, tb);
        Py_XDECREF(tmp_tb);
#endif
    }
bad:
    Py_XDECREF(owned_instance);
    return;
}

/* TupleAndListFromArray (used by fastcall) */
#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL
static CYTHON_INLINE PyObject *
__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n)
{
    PyObject *res;
    Py_ssize_t i;
    if (n <= 0) {
        return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple);
    }
    res = PyTuple_New(n);
    if (unlikely(res == NULL)) return NULL;
    for (i = 0; i < n; i++) {
        Py_INCREF(src[i]);
        if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) {
            Py_DECREF(res);
            return NULL;
        }
    }
    return res;
}
#elif CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) {
    PyObject *v;
    Py_ssize_t i;
    for (i = 0; i < length; i++) {
        v = dest[i] = src[i];
        Py_INCREF(v);
    }
}
static CYTHON_INLINE PyObject *
__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n)
{
    PyObject *res;
    if (n <= 0) {
        return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple);
    }
    res = PyTuple_New(n);
    if (unlikely(res == NULL)) return NULL;
    __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n);
    return res;
}
static CYTHON_INLINE PyObject *
__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n)
{
    PyObject *res;
    if (n <= 0) {
        return PyList_New(0);
    }
    res = PyList_New(n);
    if (unlikely(res == NULL)) return NULL;
    __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n);
    return res;
}
#endif

/* BytesEquals (used by UnicodeEquals) */
static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) {
#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\
        !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS)
    return PyObject_RichCompareBool(s1, s2, equals);
#else
    if (s1 == s2) {
        return (equals == Py_EQ);
    } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) {
        const char *ps1, *ps2;
        Py_ssize_t length = PyBytes_GET_SIZE(s1);
        if (length != PyBytes_GET_SIZE(s2))
            return (equals == Py_NE);
        ps1 = PyBytes_AS_STRING(s1);
        ps2 = PyBytes_AS_STRING(s2);
        if (ps1[0] != ps2[0]) {
            return (equals == Py_NE);
        } else if (length == 1) {
            return (equals == Py_EQ);
        } else {
            int result;
#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000)
            Py_hash_t hash1, hash2;
            hash1 = ((PyBytesObject*)s1)->ob_shash;
            hash2 = ((PyBytesObject*)s2)->ob_shash;
            if (hash1 != hash2 && hash1 != -1 && hash2 != -1) {
                return (equals == Py_NE);
            }
#endif
            result = memcmp(ps1, ps2, (size_t)length);
            return (equals == Py_EQ) ? (result == 0) : (result != 0);
        }
    } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) {
        return (equals == Py_NE);
    } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) {
        return (equals == Py_NE);
    } else {
        int result;
        PyObject* py_result = PyObject_RichCompare(s1, s2, equals);
        if (!py_result)
            return -1;
        result = __Pyx_PyObject_IsTrue(py_result);
        Py_DECREF(py_result);
        return result;
    }
#endif
}

/* UnicodeEquals (used by fastcall) */
static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) {
#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL
    return PyObject_RichCompareBool(s1, s2, equals);
#else
    int s1_is_unicode, s2_is_unicode;
    if (s1 == s2) {
        goto return_eq;
    }
    s1_is_unicode = PyUnicode_CheckExact(s1);
    s2_is_unicode = PyUnicode_CheckExact(s2);
    if (s1_is_unicode & s2_is_unicode) {
        Py_ssize_t length, length2;
        int kind;
        void *data1, *data2;
        #if !CYTHON_COMPILING_IN_LIMITED_API
        if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0))
            return -1;
        #endif
        length = __Pyx_PyUnicode_GET_LENGTH(s1);
        #if !CYTHON_ASSUME_SAFE_SIZE
        if (unlikely(length < 0)) return -1;
        #endif
        length2 = __Pyx_PyUnicode_GET_LENGTH(s2);
        #if !CYTHON_ASSUME_SAFE_SIZE
        if (unlikely(length2 < 0)) return -1;
        #endif
        if (length != length2) {
            goto return_ne;
        }
#if CYTHON_USE_UNICODE_INTERNALS
        {
            Py_hash_t hash1, hash2;
            hash1 = ((PyASCIIObject*)s1)->hash;
            hash2 = ((PyASCIIObject*)s2)->hash;
            if (hash1 != hash2 && hash1 != -1 && hash2 != -1) {
                goto return_ne;
            }
        }
#endif
        kind = __Pyx_PyUnicode_KIND(s1);
        if (kind != __Pyx_PyUnicode_KIND(s2)) {
            goto return_ne;
        }
        data1 = __Pyx_PyUnicode_DATA(s1);
        data2 = __Pyx_PyUnicode_DATA(s2);
        if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) {
            goto return_ne;
        } else if (length == 1) {
            goto return_eq;
        } else {
            int result = memcmp(data1, data2, (size_t)(length * kind));
            return (equals == Py_EQ) ? (result == 0) : (result != 0);
        }
    } else if ((s1 == Py_None) & s2_is_unicode) {
        goto return_ne;
    } else if ((s2 == Py_None) & s1_is_unicode) {
        goto return_ne;
    } else {
        int result;
        PyObject* py_result = PyObject_RichCompare(s1, s2, equals);
        if (!py_result)
            return -1;
        result = __Pyx_PyObject_IsTrue(py_result);
        Py_DECREF(py_result);
        return result;
    }
return_eq:
    return (equals == Py_EQ);
return_ne:
    return (equals == Py_NE);
#endif
}

/* fastcall */
#if CYTHON_METH_FASTCALL
static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s)
{
    Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames);
    #if !CYTHON_ASSUME_SAFE_SIZE
    if (unlikely(n == -1)) return NULL;
    #endif
    for (i = 0; i < n; i++)
    {
        PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i);
        #if !CYTHON_ASSUME_SAFE_MACROS
        if (unlikely(!namei)) return NULL;
        #endif
        if (s == namei) return kwvalues[i];
    }
    for (i = 0; i < n; i++)
    {
        PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i);
        #if !CYTHON_ASSUME_SAFE_MACROS
        if (unlikely(!namei)) return NULL;
        #endif
        int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ);
        if (unlikely(eq != 0)) {
            if (unlikely(eq < 0)) return NULL;
            return kwvalues[i];
        }
    }
    return NULL;
}
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API
CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) {
    Py_ssize_t i, nkwargs;
    PyObject *dict;
#if !CYTHON_ASSUME_SAFE_SIZE
    nkwargs = PyTuple_Size(kwnames);
    if (unlikely(nkwargs < 0)) return NULL;
#else
    nkwargs = PyTuple_GET_SIZE(kwnames);
#endif
    dict = PyDict_New();
    if (unlikely(!dict))
        return NULL;
    for (i=0; i<nkwargs; i++) {
#if !CYTHON_ASSUME_SAFE_MACROS
        PyObject *key = PyTuple_GetItem(kwnames, i);
        if (!key) goto bad;
#else
        PyObject *key = PyTuple_GET_ITEM(kwnames, i);
#endif
        if (unlikely(PyDict_SetItem(dict, key, kwvalues[i]) < 0))
            goto bad;
    }
    return dict;
bad:
    Py_DECREF(dict);
    return NULL;
}
#endif
#endif

/* RaiseArgTupleInvalid */
static void __Pyx_RaiseArgtupleInvalid(
    const char* func_name,
    int exact,
    Py_ssize_t num_min,
    Py_ssize_t num_max,
    Py_ssize_t num_found)
{
    Py_ssize_t num_expected;
    const char *more_or_less;
    if (num_found < num_min) {
        num_expected = num_min;
        more_or_less = "at least";
    } else {
        num_expected = num_max;
        more_or_less = "at most";
    }
    if (exact) {
        more_or_less = "exactly";
    }
    PyErr_Format(PyExc_TypeError,
                 "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)",
                 func_name, more_or_less, num_expected,
                 (num_expected == 1) ? "" : "s", num_found);
}

/* PyObjectCallOneArg (used by CallUnboundCMethod0) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) {
    PyObject *args[2] = {NULL, arg};
    return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
}

/* UnpackUnboundCMethod (used by CallUnboundCMethod0) */
#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000
static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) {
    PyObject *result;
    PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args));
    if (unlikely(!selfless_args)) return NULL;
    result = PyObject_Call(method, selfless_args, kwargs);
    Py_DECREF(selfless_args);
    return result;
}
#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000
static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) {
        return _PyObject_Vectorcall
            (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames);
}
#else
static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {
    return
#if PY_VERSION_HEX < 0x03090000
    _PyObject_Vectorcall
#else
    PyObject_Vectorcall
#endif
        (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames);
}
#endif
static PyMethodDef __Pyx_UnboundCMethod_Def = {
     "CythonUnboundCMethod",
     __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall),
#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000
     METH_VARARGS | METH_KEYWORDS,
#else
     METH_FASTCALL | METH_KEYWORDS,
#endif
     NULL
};
static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) {
    PyObject *method, *result=NULL;
    method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name);
    if (unlikely(!method))
        return -1;
    result = method;
#if CYTHON_COMPILING_IN_CPYTHON
    if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type)))
    {
        PyMethodDescrObject *descr = (PyMethodDescrObject*) method;
        target->func = descr->d_method->ml_meth;
        target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS);
    } else
#endif
#if CYTHON_COMPILING_IN_PYPY
#else
    if (PyCFunction_Check(method))
#endif
    {
        PyObject *self;
        int self_found;
#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY
        self = PyObject_GetAttrString(method, "__self__");
        if (!self) {
            PyErr_Clear();
        }
#else
        self = PyCFunction_GET_SELF(method);
#endif
        self_found = (self && self != Py_None);
#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY
        Py_XDECREF(self);
#endif
        if (self_found) {
            PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method);
            if (unlikely(!unbound_method)) return -1;
            Py_DECREF(method);
            result = unbound_method;
        }
    }
#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    if (unlikely(target->method)) {
        Py_DECREF(result);
    } else
#endif
    target->method = result;
    return 0;
}

/* CallUnboundCMethod0 */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) {
    int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc);
    if (likely(was_initialized == 2 && cfunc->func)) {
        if (likely(cfunc->flag == METH_NOARGS))
            return __Pyx_CallCFunction(cfunc, self, NULL);
        if (likely(cfunc->flag == METH_FASTCALL))
            return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0);
        if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS))
            return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL);
        if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS)))
            return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL);
        if (cfunc->flag == METH_VARARGS)
            return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple);
        return __Pyx__CallUnboundCMethod0(cfunc, self);
    }
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    else if (unlikely(was_initialized == 1)) {
        __Pyx_CachedCFunction tmp_cfunc = {
#ifndef __cplusplus
            0
#endif
        };
        tmp_cfunc.type = cfunc->type;
        tmp_cfunc.method_name = cfunc->method_name;
        return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self);
    }
#endif
    PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self);
    __Pyx_CachedCFunction_SetFinishedInitializing(cfunc);
    return result;
}
#endif
static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) {
    PyObject *result;
    if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL;
    result = __Pyx_PyObject_CallOneArg(cfunc->method, self);
    return result;
}

/* py_dict_items (used by OwnedDictNext) */
static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) {
    return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d);
}

/* py_dict_values (used by OwnedDictNext) */
static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) {
    return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d);
}

/* OwnedDictNext (used by RejectKeywords) */
#if CYTHON_AVOID_BORROWED_REFS
static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) {
    PyObject *next = NULL;
    if (!*ppos) {
        if (pvalue) {
            PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p);
            if (unlikely(!dictview)) goto bad;
            *ppos = PyObject_GetIter(dictview);
            Py_DECREF(dictview);
        } else {
            *ppos = PyObject_GetIter(p);
        }
        if (unlikely(!*ppos)) goto bad;
    }
    next = PyIter_Next(*ppos);
    if (!next) {
        if (PyErr_Occurred()) goto bad;
        return 0;
    }
    if (pkey && pvalue) {
        *pkey = __Pyx_PySequence_ITEM(next, 0);
        if (unlikely(*pkey)) goto bad;
        *pvalue = __Pyx_PySequence_ITEM(next, 1);
        if (unlikely(*pvalue)) goto bad;
        Py_DECREF(next);
    } else if (pkey) {
        *pkey = next;
    } else {
        assert(pvalue);
        *pvalue = next;
    }
    return 1;
  bad:
    Py_XDECREF(next);
#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000
    PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef");
#else
    PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef);
#endif
    if (pkey) *pkey = NULL;
    if (pvalue) *pvalue = NULL;
    return 0;
}
#else // !CYTHON_AVOID_BORROWED_REFS
static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) {
    int result = PyDict_Next(p, ppos, pkey, pvalue);
    if (likely(result == 1)) {
        if (pkey) Py_INCREF(*pkey);
        if (pvalue) Py_INCREF(*pvalue);
    }
    return result;
}
#endif

/* RejectKeywords */
static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds) {
    PyObject *key = NULL;
    if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) {
        key = __Pyx_PySequence_ITEM(kwds, 0);
    } else {
#if CYTHON_AVOID_BORROWED_REFS
        PyObject *pos = NULL;
#else
        Py_ssize_t pos = 0;
#endif
#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments)
        if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return;
#endif
        __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL);
#if CYTHON_AVOID_BORROWED_REFS
        Py_XDECREF(pos);
#endif
    }
    if (likely(key)) {
        PyErr_Format(PyExc_TypeError,
            "%s() got an unexpected keyword argument '%U'",
            function_name, key);
        Py_DECREF(key);
    }
}

/* GetTopmostException (used by SaveResetException) */
#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE
static _PyErr_StackItem *
__Pyx_PyErr_GetTopmostException(PyThreadState *tstate)
{
    _PyErr_StackItem *exc_info = tstate->exc_info;
    while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) &&
           exc_info->previous_item != NULL)
    {
        exc_info = exc_info->previous_item;
    }
    return exc_info;
}
#endif

/* SaveResetException */
#if CYTHON_FAST_THREAD_STATE
static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
  #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4
    _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate);
    PyObject *exc_value = exc_info->exc_value;
    if (exc_value == NULL || exc_value == Py_None) {
        *value = NULL;
        *type = NULL;
        *tb = NULL;
    } else {
        *value = exc_value;
        Py_INCREF(*value);
        *type = (PyObject*) Py_TYPE(exc_value);
        Py_INCREF(*type);
        *tb = PyException_GetTraceback(exc_value);
    }
  #elif CYTHON_USE_EXC_INFO_STACK
    _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate);
    *type = exc_info->exc_type;
    *value = exc_info->exc_value;
    *tb = exc_info->exc_traceback;
    Py_XINCREF(*type);
    Py_XINCREF(*value);
    Py_XINCREF(*tb);
  #else
    *type = tstate->exc_type;
    *value = tstate->exc_value;
    *tb = tstate->exc_traceback;
    Py_XINCREF(*type);
    Py_XINCREF(*value);
    Py_XINCREF(*tb);
  #endif
}
static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) {
  #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4
    _PyErr_StackItem *exc_info = tstate->exc_info;
    PyObject *tmp_value = exc_info->exc_value;
    exc_info->exc_value = value;
    Py_XDECREF(tmp_value);
    Py_XDECREF(type);
    Py_XDECREF(tb);
  #else
    PyObject *tmp_type, *tmp_value, *tmp_tb;
    #if CYTHON_USE_EXC_INFO_STACK
    _PyErr_StackItem *exc_info = tstate->exc_info;
    tmp_type = exc_info->exc_type;
    tmp_value = exc_info->exc_value;
    tmp_tb = exc_info->exc_traceback;
    exc_info->exc_type = type;
    exc_info->exc_value = value;
    exc_info->exc_traceback = tb;
    #else
    tmp_type = tstate->exc_type;
    tmp_value = tstate->exc_value;
    tmp_tb = tstate->exc_traceback;
    tstate->exc_type = type;
    tstate->exc_value = value;
    tstate->exc_traceback = tb;
    #endif
    Py_XDECREF(tmp_type);
    Py_XDECREF(tmp_value);
    Py_XDECREF(tmp_tb);
  #endif
}
#endif

/* GetException */
#if CYTHON_FAST_THREAD_STATE
static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb)
#else
static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb)
#endif
{
    PyObject *local_type = NULL, *local_value, *local_tb = NULL;
#if CYTHON_FAST_THREAD_STATE
    PyObject *tmp_type, *tmp_value, *tmp_tb;
  #if PY_VERSION_HEX >= 0x030C0000
    local_value = tstate->current_exception;
    tstate->current_exception = 0;
  #else
    local_type = tstate->curexc_type;
    local_value = tstate->curexc_value;
    local_tb = tstate->curexc_traceback;
    tstate->curexc_type = 0;
    tstate->curexc_value = 0;
    tstate->curexc_traceback = 0;
  #endif
#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000
    local_value = PyErr_GetRaisedException();
#else
    PyErr_Fetch(&local_type, &local_value, &local_tb);
#endif
#if __PYX_LIMITED_VERSION_HEX > 0x030C0000
    if (likely(local_value)) {
        local_type = (PyObject*) Py_TYPE(local_value);
        Py_INCREF(local_type);
        local_tb = PyException_GetTraceback(local_value);
    }
#else
    PyErr_NormalizeException(&local_type, &local_value, &local_tb);
#if CYTHON_FAST_THREAD_STATE
    if (unlikely(tstate->curexc_type))
#else
    if (unlikely(PyErr_Occurred()))
#endif
        goto bad;
    if (local_tb) {
        if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0))
            goto bad;
    }
#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000
    Py_XINCREF(local_tb);
    Py_XINCREF(local_type);
    Py_XINCREF(local_value);
    *type = local_type;
    *value = local_value;
    *tb = local_tb;
#if CYTHON_FAST_THREAD_STATE
    #if CYTHON_USE_EXC_INFO_STACK
    {
        _PyErr_StackItem *exc_info = tstate->exc_info;
      #if PY_VERSION_HEX >= 0x030B00a4
        tmp_value = exc_info->exc_value;
        exc_info->exc_value = local_value;
        tmp_type = NULL;
        tmp_tb = NULL;
        Py_XDECREF(local_type);
        Py_XDECREF(local_tb);
      #else
        tmp_type = exc_info->exc_type;
        tmp_value = exc_info->exc_value;
        tmp_tb = exc_info->exc_traceback;
        exc_info->exc_type = local_type;
        exc_info->exc_value = local_value;
        exc_info->exc_traceback = local_tb;
      #endif
    }
    #else
    tmp_type = tstate->exc_type;
    tmp_value = tstate->exc_value;
    tmp_tb = tstate->exc_traceback;
    tstate->exc_type = local_type;
    tstate->exc_value = local_value;
    tstate->exc_traceback = local_tb;
    #endif
    Py_XDECREF(tmp_type);
    Py_XDECREF(tmp_value);
    Py_XDECREF(tmp_tb);
#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000
    PyErr_SetHandledException(local_value);
    Py_XDECREF(local_value);
    Py_XDECREF(local_type);
    Py_XDECREF(local_tb);
#else
    PyErr_SetExcInfo(local_type, local_value, local_tb);
#endif
    return 0;
#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000
bad:
    *type = 0;
    *value = 0;
    *tb = 0;
    Py_XDECREF(local_type);
    Py_XDECREF(local_value);
    Py_XDECREF(local_tb);
    return -1;
#endif
}

/* PyObjectSetAttrStr */
#if CYTHON_USE_TYPE_SLOTS
static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) {
    PyTypeObject* tp = Py_TYPE(obj);
    if (likely(tp->tp_setattro))
        return tp->tp_setattro(obj, attr_name, value);
    return PyObject_SetAttr(obj, attr_name, value);
}
#endif

/* GetAttr3 */
#if __PYX_LIMITED_VERSION_HEX < 0x030d0000
static PyObject *__Pyx_GetAttr3Default(PyObject *d) {
    __Pyx_PyThreadState_declare
    __Pyx_PyThreadState_assign
    if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError)))
        return NULL;
    __Pyx_PyErr_Clear();
    Py_INCREF(d);
    return d;
}
#endif
static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) {
    PyObject *r;
#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
    int res = PyObject_GetOptionalAttr(o, n, &r);
    return (res != 0) ? r : __Pyx_NewRef(d);
#else
  #if CYTHON_USE_TYPE_SLOTS
    if (likely(PyUnicode_Check(n))) {
        r = __Pyx_PyObject_GetAttrStrNoError(o, n);
        if (unlikely(!r) && likely(!PyErr_Occurred())) {
            r = __Pyx_NewRef(d);
        }
        return r;
    }
  #endif
    r = PyObject_GetAttr(o, n);
    return (likely(r)) ? r : __Pyx_GetAttr3Default(d);
#endif
}

/* PyDictVersioning (used by GetModuleGlobalName) */
#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS
static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) {
    PyObject *dict = Py_TYPE(obj)->tp_dict;
    return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0;
}
static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) {
    PyObject **dictptr = NULL;
    Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset;
    if (offset) {
#if CYTHON_COMPILING_IN_CPYTHON
        dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj);
#else
        dictptr = _PyObject_GetDictPtr(obj);
#endif
    }
    return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0;
}
static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) {
    PyObject *dict = Py_TYPE(obj)->tp_dict;
    if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict)))
        return 0;
    return obj_dict_version == __Pyx_get_object_dict_version(obj);
}
#endif

/* GetModuleGlobalName */
#if CYTHON_USE_DICT_VERSIONS
static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value)
#else
static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name)
#endif
{
    PyObject *result;
#if CYTHON_COMPILING_IN_LIMITED_API
    if (unlikely(!__pyx_m)) {
        if (!PyErr_Occurred())
            PyErr_SetNone(PyExc_NameError);
        return NULL;
    }
    result = PyObject_GetAttr(__pyx_m, name);
    if (likely(result)) {
        return result;
    }
    PyErr_Clear();
#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
    if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear();
    __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version)
    if (likely(result)) {
        return result;
    }
#else
    result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash);
    __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version)
    if (likely(result)) {
        return __Pyx_NewRef(result);
    }
    PyErr_Clear();
#endif
    return __Pyx_GetBuiltinName(name);
}

/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */
static void __Pyx_RaiseDoubleKeywordsError(
    const char* func_name,
    PyObject* kw_name)
{
    PyErr_Format(PyExc_TypeError,
        "%s() got multiple values for keyword argument '%U'", func_name, kw_name);
}

/* CallUnboundCMethod2 */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) {
    int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc);
    if (likely(was_initialized == 2 && cfunc->func)) {
        PyObject *args[2] = {arg1, arg2};
        if (cfunc->flag == METH_FASTCALL) {
            return __Pyx_CallCFunctionFast(cfunc, self, args, 2);
        }
        if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS))
            return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL);
    }
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    else if (unlikely(was_initialized == 1)) {
        __Pyx_CachedCFunction tmp_cfunc = {
#ifndef __cplusplus
            0
#endif
        };
        tmp_cfunc.type = cfunc->type;
        tmp_cfunc.method_name = cfunc->method_name;
        return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2);
    }
#endif
    PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2);
    __Pyx_CachedCFunction_SetFinishedInitializing(cfunc);
    return result;
}
#endif
static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){
    if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL;
#if CYTHON_COMPILING_IN_CPYTHON
    if (cfunc->func && (cfunc->flag & METH_VARARGS)) {
        PyObject *result = NULL;
        PyObject *args = PyTuple_New(2);
        if (unlikely(!args)) return NULL;
        Py_INCREF(arg1);
        PyTuple_SET_ITEM(args, 0, arg1);
        Py_INCREF(arg2);
        PyTuple_SET_ITEM(args, 1, arg2);
        if (cfunc->flag & METH_KEYWORDS)
            result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL);
        else
            result = __Pyx_CallCFunction(cfunc, self, args);
        Py_DECREF(args);
        return result;
    }
#endif
    {
        PyObject *args[4] = {NULL, self, arg1, arg2};
        return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
    }
}

/* ParseKeywordsImpl (used by ParseKeywords) */
static int __Pyx_ValidateDuplicatePosArgs(
    PyObject *kwds,
    PyObject ** const argnames[],
    PyObject ** const *first_kw_arg,
    const char* function_name)
{
    PyObject ** const *name = argnames;
    while (name != first_kw_arg) {
        PyObject *key = **name;
        int found = PyDict_Contains(kwds, key);
        if (unlikely(found)) {
            if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key);
            goto bad;
        }
        name++;
    }
    return 0;
bad:
    return -1;
}
#if CYTHON_USE_UNICODE_INTERNALS
static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) {
    int kind;
    Py_ssize_t len = PyUnicode_GET_LENGTH(s1);
    if (len != PyUnicode_GET_LENGTH(s2)) return 0;
    kind = PyUnicode_KIND(s1);
    if (kind != PyUnicode_KIND(s2)) return 0;
    const void *data1 = PyUnicode_DATA(s1);
    const void *data2 = PyUnicode_DATA(s2);
    return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0);
}
#endif
static int __Pyx_MatchKeywordArg_str(
    PyObject *key,
    PyObject ** const argnames[],
    PyObject ** const *first_kw_arg,
    size_t *index_found,
    const char *function_name)
{
    PyObject ** const *name;
    #if CYTHON_USE_UNICODE_INTERNALS
    Py_hash_t key_hash = ((PyASCIIObject*)key)->hash;
    if (unlikely(key_hash == -1)) {
        key_hash = PyObject_Hash(key);
        if (unlikely(key_hash == -1))
            goto bad;
    }
    #endif
    name = first_kw_arg;
    while (*name) {
        PyObject *name_str = **name;
        #if CYTHON_USE_UNICODE_INTERNALS
        if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) {
            *index_found = (size_t) (name - argnames);
            return 1;
        }
        #else
        #if CYTHON_ASSUME_SAFE_SIZE
        if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key))
        #endif
        {
            int cmp = PyUnicode_Compare(name_str, key);
            if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad;
            if (cmp == 0) {
                *index_found = (size_t) (name - argnames);
                return 1;
            }
        }
        #endif
        name++;
    }
    name = argnames;
    while (name != first_kw_arg) {
        PyObject *name_str = **name;
        #if CYTHON_USE_UNICODE_INTERNALS
        if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) {
            if (__Pyx_UnicodeKeywordsEqual(name_str, key))
                goto arg_passed_twice;
        }
        #else
        #if CYTHON_ASSUME_SAFE_SIZE
        if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key))
        #endif
        {
            if (unlikely(name_str == key)) goto arg_passed_twice;
            int cmp = PyUnicode_Compare(name_str, key);
            if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad;
            if (cmp == 0) goto arg_passed_twice;
        }
        #endif
        name++;
    }
    return 0;
arg_passed_twice:
    __Pyx_RaiseDoubleKeywordsError(function_name, key);
    goto bad;
bad:
    return -1;
}
static int __Pyx_MatchKeywordArg_nostr(
    PyObject *key,
    PyObject ** const argnames[],
    PyObject ** const *first_kw_arg,
    size_t *index_found,
    const char *function_name)
{
    PyObject ** const *name;
    if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type;
    name = first_kw_arg;
    while (*name) {
        int cmp = PyObject_RichCompareBool(**name, key, Py_EQ);
        if (cmp == 1) {
            *index_found = (size_t) (name - argnames);
            return 1;
        }
        if (unlikely(cmp == -1)) goto bad;
        name++;
    }
    name = argnames;
    while (name != first_kw_arg) {
        int cmp = PyObject_RichCompareBool(**name, key, Py_EQ);
        if (unlikely(cmp != 0)) {
            if (cmp == 1) goto arg_passed_twice;
            else goto bad;
        }
        name++;
    }
    return 0;
arg_passed_twice:
    __Pyx_RaiseDoubleKeywordsError(function_name, key);
    goto bad;
invalid_keyword_type:
    PyErr_Format(PyExc_TypeError,
        "%.200s() keywords must be strings", function_name);
    goto bad;
bad:
    return -1;
}
static CYTHON_INLINE int __Pyx_MatchKeywordArg(
    PyObject *key,
    PyObject ** const argnames[],
    PyObject ** const *first_kw_arg,
    size_t *index_found,
    const char *function_name)
{
    return likely(PyUnicode_CheckExact(key)) ?
        __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) :
        __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name);
}
static void __Pyx_RejectUnknownKeyword(
    PyObject *kwds,
    PyObject ** const argnames[],
    PyObject ** const *first_kw_arg,
    const char *function_name)
{
    #if CYTHON_AVOID_BORROWED_REFS
    PyObject *pos = NULL;
    #else
    Py_ssize_t pos = 0;
    #endif
    PyObject *key = NULL;
    __Pyx_BEGIN_CRITICAL_SECTION(kwds);
    while (
        #if CYTHON_AVOID_BORROWED_REFS
        __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL)
        #else
        PyDict_Next(kwds, &pos, &key, NULL)
        #endif
    ) {
        PyObject** const *name = first_kw_arg;
        while (*name && (**name != key)) name++;
        if (!*name) {
            size_t index_found = 0;
            int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name);
            if (cmp != 1) {
                if (cmp == 0) {
                    PyErr_Format(PyExc_TypeError,
                        "%s() got an unexpected keyword argument '%U'",
                        function_name, key);
                }
                #if CYTHON_AVOID_BORROWED_REFS
                Py_DECREF(key);
                #endif
                break;
            }
        }
        #if CYTHON_AVOID_BORROWED_REFS
        Py_DECREF(key);
        #endif
    }
    __Pyx_END_CRITICAL_SECTION();
    #if CYTHON_AVOID_BORROWED_REFS
    Py_XDECREF(pos);
    #endif
    assert(PyErr_Occurred());
}
static int __Pyx_ParseKeywordDict(
    PyObject *kwds,
    PyObject ** const argnames[],
    PyObject *values[],
    Py_ssize_t num_pos_args,
    Py_ssize_t num_kwargs,
    const char* function_name,
    int ignore_unknown_kwargs)
{
    PyObject** const *name;
    PyObject** const *first_kw_arg = argnames + num_pos_args;
    Py_ssize_t extracted = 0;
#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments)
    if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1;
#endif
    name = first_kw_arg;
    while (*name && num_kwargs > extracted) {
        PyObject * key = **name;
        PyObject *value;
        int found = 0;
        #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
        found = PyDict_GetItemRef(kwds, key, &value);
        #else
        value = PyDict_GetItemWithError(kwds, key);
        if (value) {
            Py_INCREF(value);
            found = 1;
        } else {
            if (unlikely(PyErr_Occurred())) goto bad;
        }
        #endif
        if (found) {
            if (unlikely(found < 0)) goto bad;
            values[name-argnames] = value;
            extracted++;
        }
        name++;
    }
    if (num_kwargs > extracted) {
        if (ignore_unknown_kwargs) {
            if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1))
                goto bad;
        } else {
            __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name);
            goto bad;
        }
    }
    return 0;
bad:
    return -1;
}
static int __Pyx_ParseKeywordDictToDict(
    PyObject *kwds,
    PyObject ** const argnames[],
    PyObject *kwds2,
    PyObject *values[],
    Py_ssize_t num_pos_args,
    const char* function_name)
{
    PyObject** const *name;
    PyObject** const *first_kw_arg = argnames + num_pos_args;
    Py_ssize_t len;
#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments)
    if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1;
#endif
    if (PyDict_Update(kwds2, kwds) < 0) goto bad;
    name = first_kw_arg;
    while (*name) {
        PyObject *key = **name;
        PyObject *value;
#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop))
        int found = PyDict_Pop(kwds2, key, &value);
        if (found) {
            if (unlikely(found < 0)) goto bad;
            values[name-argnames] = value;
        }
#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000
        int found = PyDict_GetItemRef(kwds2, key, &value);
        if (found) {
            if (unlikely(found < 0)) goto bad;
            values[name-argnames] = value;
            if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad;
        }
#else
    #if CYTHON_COMPILING_IN_CPYTHON
        value = _PyDict_Pop(kwds2, key, kwds2);
    #else
        value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2);
    #endif
        if (value == kwds2) {
            Py_DECREF(value);
        } else {
            if (unlikely(!value)) goto bad;
            values[name-argnames] = value;
        }
#endif
        name++;
    }
    len = PyDict_Size(kwds2);
    if (len > 0) {
        return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name);
    } else if (unlikely(len == -1)) {
        goto bad;
    }
    return 0;
bad:
    return -1;
}
static int __Pyx_ParseKeywordsTuple(
    PyObject *kwds,
    PyObject * const *kwvalues,
    PyObject ** const argnames[],
    PyObject *kwds2,
    PyObject *values[],
    Py_ssize_t num_pos_args,
    Py_ssize_t num_kwargs,
    const char* function_name,
    int ignore_unknown_kwargs)
{
    PyObject *key = NULL;
    PyObject** const * name;
    PyObject** const *first_kw_arg = argnames + num_pos_args;
    for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) {
#if CYTHON_AVOID_BORROWED_REFS
        key = __Pyx_PySequence_ITEM(kwds, pos);
#else
        key = __Pyx_PyTuple_GET_ITEM(kwds, pos);
#endif
#if !CYTHON_ASSUME_SAFE_MACROS
        if (unlikely(!key)) goto bad;
#endif
        name = first_kw_arg;
        while (*name && (**name != key)) name++;
        if (*name) {
            PyObject *value = kwvalues[pos];
            values[name-argnames] = __Pyx_NewRef(value);
        } else {
            size_t index_found = 0;
            int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name);
            if (cmp == 1) {
                PyObject *value = kwvalues[pos];
                values[index_found] = __Pyx_NewRef(value);
            } else {
                if (unlikely(cmp == -1)) goto bad;
                if (kwds2) {
                    PyObject *value = kwvalues[pos];
                    if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad;
                } else if (!ignore_unknown_kwargs) {
                    goto invalid_keyword;
                }
            }
        }
        #if CYTHON_AVOID_BORROWED_REFS
        Py_DECREF(key);
        key = NULL;
        #endif
    }
    return 0;
invalid_keyword:
    PyErr_Format(PyExc_TypeError,
        "%s() got an unexpected keyword argument '%U'",
        function_name, key);
    goto bad;
bad:
    #if CYTHON_AVOID_BORROWED_REFS
    Py_XDECREF(key);
    #endif
    return -1;
}

/* ParseKeywords */
static int __Pyx_ParseKeywords(
    PyObject *kwds,
    PyObject * const *kwvalues,
    PyObject ** const argnames[],
    PyObject *kwds2,
    PyObject *values[],
    Py_ssize_t num_pos_args,
    Py_ssize_t num_kwargs,
    const char* function_name,
    int ignore_unknown_kwargs)
{
    if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds)))
        return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs);
    else if (kwds2)
        return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name);
    else
        return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs);
}

/* RaiseUnexpectedTypeError */
static int
__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj)
{
    __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj));
    PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME,
                 expected, obj_type_name);
    __Pyx_DECREF_TypeName(obj_type_name);
    return 0;
}

/* ImportImpl (used by Import) */
static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) {
    PyObject *imported_module = PyImport_GetModule(qualname);
    if (unlikely(!imported_module)) {
        *module = NULL;
        if (PyErr_Occurred()) {
            return -1;
        }
        return 0;
    }
    *module = imported_module;
    return 1;
}
static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) {
    PyObject *imported_module;
    PyObject *top_level_package_name;
    Py_ssize_t i;
    int status, module_found;
    Py_ssize_t dot_index;
    module_found = __Pyx__Import_GetModule(qualname, &imported_module);
    if (unlikely(!module_found || module_found == -1)) {
        *module = NULL;
        return module_found;
    }
    if (imported_names) {
        for (i = 0; i < len_imported_names; i++) {
            PyObject *imported_name = imported_names[i];
#if __PYX_LIMITED_VERSION_HEX < 0x030d0000
            int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name);
#else
            int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name);
            if (unlikely(has_imported_attribute == -1)) goto error;
#endif
            if (!has_imported_attribute) {
                goto not_found;
            }
        }
        *module = imported_module;
        return 1;
    }
    dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1);
    if (dot_index == -1) {
        *module = imported_module;
        return 1;
    }
    if (unlikely(dot_index == -2)) goto error;
    top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index);
    if (unlikely(!top_level_package_name)) goto error;
    Py_DECREF(imported_module);
    status = __Pyx__Import_GetModule(top_level_package_name, module);
    Py_DECREF(top_level_package_name);
    return status;
error:
    Py_DECREF(imported_module);
    *module = NULL;
    return -1;
not_found:
    Py_DECREF(imported_module);
    *module = NULL;
    return 0;
}
static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) {
    PyObject *module = 0;
    PyObject *empty_dict = 0;
    PyObject *from_list = 0;
    int module_found;
    if (!qualname) {
        qualname = name;
    }
    module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module);
    if (likely(module_found == 1)) {
        return module;
    } else if (unlikely(module_found == -1)) {
        return NULL;
    }
    empty_dict = PyDict_New();
    if (unlikely(!empty_dict))
        goto bad;
    if (imported_names) {
#if CYTHON_COMPILING_IN_CPYTHON
        from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names);
        if (unlikely(!from_list))
            goto bad;
#else
        from_list = PyList_New(len_imported_names);
        if (unlikely(!from_list)) goto bad;
        for (Py_ssize_t i=0; i<len_imported_names; ++i) {
            if (PyList_SetItem(from_list, i, __Pyx_NewRef(imported_names[i])) < 0) goto bad;
        }
#endif
    }
    if (level == -1) {
        const char* package_sep = strchr(__Pyx_MODULE_NAME, '.');
        if (package_sep != (0)) {
            module = PyImport_ImportModuleLevelObject(
                name, moddict, empty_dict, from_list, 1);
            if (unlikely(!module)) {
                if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError)))
                    goto bad;
                PyErr_Clear();
            }
        }
        level = 0;
    }
    if (!module) {
        module = PyImport_ImportModuleLevelObject(
            name, moddict, empty_dict, from_list, level);
    }
bad:
    Py_XDECREF(from_list);
    Py_XDECREF(empty_dict);
    return module;
}

/* Import */
static PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level) {
    return __Pyx__Import(name, imported_names, len_imported_names, qualname, __pyx_mstate_global->__pyx_d, level);
}

/* ImportFrom */
static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) {
    PyObject* value = __Pyx_PyObject_GetAttrStr(module, name);
    if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) {
        const char* module_name_str = 0;
        PyObject* module_name = 0;
        PyObject* module_dot = 0;
        PyObject* full_name = 0;
        PyErr_Clear();
        module_name_str = PyModule_GetName(module);
        if (unlikely(!module_name_str)) { goto modbad; }
        module_name = PyUnicode_FromString(module_name_str);
        if (unlikely(!module_name)) { goto modbad; }
        module_dot = PyUnicode_Concat(module_name, __pyx_mstate_global->__pyx_kp_u__2);
        if (unlikely(!module_dot)) { goto modbad; }
        full_name = PyUnicode_Concat(module_dot, name);
        if (unlikely(!full_name)) { goto modbad; }
        #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM  < 0x07030400) ||\
                CYTHON_COMPILING_IN_GRAAL
        {
            PyObject *modules = PyImport_GetModuleDict();
            if (unlikely(!modules))
                goto modbad;
            value = PyObject_GetItem(modules, full_name);
        }
        #else
        value = PyImport_GetModule(full_name);
        #endif
      modbad:
        Py_XDECREF(full_name);
        Py_XDECREF(module_dot);
        Py_XDECREF(module_name);
    }
    if (unlikely(!value)) {
        PyErr_Format(PyExc_ImportError, "cannot import name %S", name);
    }
    return value;
}

/* SetItemInt */
static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) {
    int r;
    if (unlikely(!j)) return -1;
    r = PyObject_SetItem(o, j, v);
    Py_DECREF(j);
    return r;
}
static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list,
                                               int wraparound, int boundscheck, int unsafe_shared) {
    CYTHON_MAYBE_UNUSED_VAR(unsafe_shared);
#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS
    if (is_list || PyList_CheckExact(o)) {
        Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o));
        if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) {
            Py_INCREF(v);
            return PyList_SetItem(o, n, v);
        } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) {
            PyObject* old;
            Py_INCREF(v);
            old = PyList_GET_ITEM(o, n);
            PyList_SET_ITEM(o, n, v);
            Py_DECREF(old);
            return 0;
        }
    } else
#endif
#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY
    {
        PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping;
        PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence;
        if (!is_list && mm && mm->mp_ass_subscript) {
            int r;
            PyObject *key = PyLong_FromSsize_t(i);
            if (unlikely(!key)) return -1;
            r = mm->mp_ass_subscript(o, key, v);
            Py_DECREF(key);
            return r;
        }
        if (is_list || likely(sm && sm->sq_ass_item)) {
            if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) {
                Py_ssize_t l = sm->sq_length(o);
                if (likely(l >= 0)) {
                    i += l;
                } else {
                    if (!PyErr_ExceptionMatches(PyExc_OverflowError))
                        return -1;
                    PyErr_Clear();
                }
            }
            return sm->sq_ass_item(o, i, v);
        }
    }
#else
    if (is_list || !PyMapping_Check(o)) {
        return PySequence_SetItem(o, i, v);
    }
#endif
    (void)wraparound;
    (void)boundscheck;
    return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v);
}

/* PyObjectVectorCallKwBuilder */
#if CYTHON_VECTORCALL
static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) {
    (void)__Pyx_PyObject_FastCallDict;
    Py_INCREF(key);
    if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1;
    args[n] = value;
    return 0;
}
CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) {
    (void)__Pyx_VectorcallBuilder_AddArgStr;
    if (unlikely(!PyUnicode_Check(key))) {
        PyErr_SetString(PyExc_TypeError, "keywords must be strings");
        return -1;
    }
    return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n);
}
static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) {
    PyObject *pyKey = PyUnicode_FromString(key);
    if (!pyKey) return -1;
    return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n);
}
#else // CYTHON_VECTORCALL
CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) {
    if (unlikely(!PyUnicode_Check(key))) {
        PyErr_SetString(PyExc_TypeError, "keywords must be strings");
        return -1;
    }
    return PyDict_SetItem(builder, key, value);
}
#endif

/* PyUnicode_Unicode */
static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj) {
    if (unlikely(obj == Py_None))
        obj = __pyx_mstate_global->__pyx_kp_u_None;
    return __Pyx_NewRef(obj);
}

/* SetStringIndexingError (used by GetItemIntUnicode) */
static void __Pyx_SetStringIndexingError(const char* message, int has_gil) {
    if (!has_gil) {
        PyGILState_STATE gil_state = PyGILState_Ensure();
        PyErr_SetString(PyExc_IndexError, message);
        PyGILState_Release(gil_state);
    } else
        PyErr_SetString(PyExc_IndexError, message);
}

/* GetItemIntUnicode */
static CYTHON_INLINE Py_UCS4 __Pyx_GetItemInt_Unicode_Fast(PyObject* ustring, Py_ssize_t i,
                                                           int wraparound, int boundscheck, int has_gil) {
    Py_ssize_t length;
    if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return (Py_UCS4)-1;
    if (wraparound | boundscheck) {
        length = __Pyx_PyUnicode_GET_LENGTH(ustring);
        #if !CYTHON_ASSUME_SAFE_SIZE
        if (unlikely(length < 0)) return (Py_UCS4)-1;
        #endif
        if (wraparound & unlikely(i < 0)) i += length;
        if ((!boundscheck) || likely(__Pyx_is_valid_index(i, length))) {
            return __Pyx_PyUnicode_READ_CHAR(ustring, i);
        } else {
            __Pyx_SetStringIndexingError("string index out of range", has_gil);
            return (Py_UCS4)-1;
        }
    } else {
        return __Pyx_PyUnicode_READ_CHAR(ustring, i);
    }
}

/* UnicodeConcatInPlace */
# if CYTHON_COMPILING_IN_CPYTHON
static int
__Pyx_unicode_modifiable(PyObject *unicode, int unsafe_shared)
{
    if (!__Pyx_IS_UNIQUELY_REFERENCED(unicode, unsafe_shared))
        return 0;
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030F0000
    if (PyUnstable_Unicode_GET_CACHED_HASH(unicode) != -1)
        return 0;
#endif
    if (!PyUnicode_CheckExact(unicode))
        return 0;
    if (PyUnicode_CHECK_INTERNED(unicode))
        return 0;
    return 1;
}
static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right, int unsafe_shared
        #if CYTHON_REFNANNY
        , void* __pyx_refnanny
        #endif
    ) {
    PyObject *left = *p_left;
    Py_ssize_t left_len, right_len, new_len;
    if (unlikely(__Pyx_PyUnicode_READY(left) == -1))
        return NULL;
    if (unlikely(__Pyx_PyUnicode_READY(right) == -1))
        return NULL;
    left_len = PyUnicode_GET_LENGTH(left);
    if (left_len == 0) {
        Py_INCREF(right);
        return right;
    }
    right_len = PyUnicode_GET_LENGTH(right);
    if (right_len == 0) {
        Py_INCREF(left);
        return left;
    }
    if (unlikely(left_len > PY_SSIZE_T_MAX - right_len)) {
        PyErr_SetString(PyExc_OverflowError,
                        "strings are too large to concat");
        return NULL;
    }
    new_len = left_len + right_len;
    if (__Pyx_unicode_modifiable(left, unsafe_shared)
            && PyUnicode_CheckExact(right)
            && PyUnicode_KIND(right) <= PyUnicode_KIND(left)
            && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) {
        int ret;
        __Pyx_GIVEREF(*p_left);
        ret = PyUnicode_Resize(p_left, new_len);
        __Pyx_GOTREF(*p_left);
        if (unlikely(ret != 0))
            return NULL;
        #if PY_VERSION_HEX >= 0x030d0000
        if (unlikely(PyUnicode_CopyCharacters(*p_left, left_len, right, 0, right_len) < 0)) return NULL;
        #else
        _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len);
        #endif
        __Pyx_INCREF(*p_left);
        __Pyx_GIVEREF(*p_left);
        return *p_left;
    } else {
        return __Pyx_PyUnicode_Concat(left, right);
    }
  }
#endif

/* PyUnicode_Substring */
static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Substring(
            PyObject* text, Py_ssize_t start, Py_ssize_t stop) {
    Py_ssize_t length;
    #if !CYTHON_COMPILING_IN_LIMITED_API
    if (unlikely(__Pyx_PyUnicode_READY(text) == -1)) return NULL;
    #endif
    length = __Pyx_PyUnicode_GET_LENGTH(text);
    #if !CYTHON_ASSUME_SAFE_SIZE
    if (unlikely(length < 0)) return NULL;
    #endif
    if (start < 0) {
        start += length;
        if (start < 0)
            start = 0;
    }
    if (stop < 0)
        stop += length;
    else if (stop > length)
        stop = length;
    if (stop <= start)
        return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_unicode);
    if (start == 0 && stop == length)
        return __Pyx_NewRef(text);
#if CYTHON_COMPILING_IN_LIMITED_API
    return PyUnicode_Substring(text, start, stop);
#else
    return PyUnicode_FromKindAndData(PyUnicode_KIND(text),
        PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text), stop-start);
#endif
}

/* DictGetItem */
#if !CYTHON_COMPILING_IN_PYPY
static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) {
    PyObject *value;
    if (unlikely(__Pyx_PyDict_GetItemRef(d, key, &value) == 0)) { // no value, no error
        if (unlikely(PyTuple_Check(key))) {
            PyObject* args = PyTuple_Pack(1, key);
            if (likely(args)) {
                PyErr_SetObject(PyExc_KeyError, args);
                Py_DECREF(args);
            }
        } else {
            PyErr_SetObject(PyExc_KeyError, key);
        }
    }
    return value;
}
#endif

/* PyObjectLookupSpecial */
#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS
static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) {
    PyObject *res;
    PyTypeObject *tp = Py_TYPE(obj);
    res = _PyType_Lookup(tp, attr_name);
    if (likely(res)) {
        descrgetfunc f = Py_TYPE(res)->tp_descr_get;
        if (!f) {
            Py_INCREF(res);
        } else {
            res = f(res, obj, (PyObject *)tp);
        }
    } else if (with_error) {
        PyErr_SetObject(PyExc_AttributeError, attr_name);
    }
    return res;
}
#endif

/* RaiseUnboundLocalError */
static void __Pyx_RaiseUnboundLocalError(const char *varname) {
    PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname);
}

/* PyObjectVectorCallMethodKwBuilder */
#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000
static PyObject *__Pyx_Object_VectorcallMethod_CallFromBuilder(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames) {
    PyObject *result;
    PyObject *obj = PyObject_GetAttr(args[0], name);
    if (unlikely(!obj))
        return NULL;
    result = __Pyx_Object_Vectorcall_CallFromBuilder(obj, args+1, nargsf-1, kwnames);
    Py_DECREF(obj);
    return result;
}
#endif

/* FastTypeChecks */
#if CYTHON_COMPILING_IN_CPYTHON
static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) {
    while (a) {
        a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*);
        if (a == b)
            return 1;
    }
    return b == &PyBaseObject_Type;
}
static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) {
    PyObject *mro;
    if (a == b) return 1;
    mro = a->tp_mro;
    if (likely(mro)) {
        Py_ssize_t i, n;
        n = PyTuple_GET_SIZE(mro);
        for (i = 0; i < n; i++) {
            if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b)
                return 1;
        }
        return 0;
    }
    return __Pyx_InBases(a, b);
}
static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) {
    PyObject *mro;
    if (cls == a || cls == b) return 1;
    mro = cls->tp_mro;
    if (likely(mro)) {
        Py_ssize_t i, n;
        n = PyTuple_GET_SIZE(mro);
        for (i = 0; i < n; i++) {
            PyObject *base = PyTuple_GET_ITEM(mro, i);
            if (base == (PyObject *)a || base == (PyObject *)b)
                return 1;
        }
        return 0;
    }
    return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b);
}
static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) {
    if (exc_type1) {
        return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2);
    } else {
        return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2);
    }
}
static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) {
    Py_ssize_t i, n;
    assert(PyExceptionClass_Check(exc_type));
    n = PyTuple_GET_SIZE(tuple);
    for (i=0; i<n; i++) {
        if (exc_type == PyTuple_GET_ITEM(tuple, i)) return 1;
    }
    for (i=0; i<n; i++) {
        PyObject *t = PyTuple_GET_ITEM(tuple, i);
        if (likely(PyExceptionClass_Check(t))) {
            if (__Pyx_inner_PyErr_GivenExceptionMatches2(exc_type, NULL, t)) return 1;
        } else {
        }
    }
    return 0;
}
static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) {
    if (likely(err == exc_type)) return 1;
    if (likely(PyExceptionClass_Check(err))) {
        if (likely(PyExceptionClass_Check(exc_type))) {
            return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type);
        } else if (likely(PyTuple_Check(exc_type))) {
            return __Pyx_PyErr_GivenExceptionMatchesTuple(err, exc_type);
        } else {
        }
    }
    return PyErr_GivenExceptionMatches(err, exc_type);
}
static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) {
    assert(PyExceptionClass_Check(exc_type1));
    assert(PyExceptionClass_Check(exc_type2));
    if (likely(err == exc_type1 || err == exc_type2)) return 1;
    if (likely(PyExceptionClass_Check(err))) {
        return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2);
    }
    return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2));
}
#endif

/* SliceTupleAndList */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop, Py_ssize_t* _length) {
    Py_ssize_t start = *_start, stop = *_stop, length = *_length;
    if (start < 0) {
        start += length;
        if (start < 0)
            start = 0;
    }
    if (stop < 0)
        stop += length;
    else if (stop > length)
        stop = length;
    *_length = stop - start;
    *_start = start;
    *_stop = stop;
}
static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice(
            PyObject* src, Py_ssize_t start, Py_ssize_t stop) {
    Py_ssize_t length = PyTuple_GET_SIZE(src);
    __Pyx_crop_slice(&start, &stop, &length);
    return __Pyx_PyTuple_FromArray(((PyTupleObject*)src)->ob_item + start, length);
}
static CYTHON_INLINE PyObject* __Pyx_PyList_GetSlice_locked(
            PyObject* src, Py_ssize_t start, Py_ssize_t stop) {
    Py_ssize_t length = PyList_GET_SIZE(src);
    __Pyx_crop_slice(&start, &stop, &length);
    if (length <= 0) {
        return PyList_New(0);
    }
    return __Pyx_PyList_FromArray(((PyListObject*)src)->ob_item + start, length);
}
static CYTHON_INLINE PyObject* __Pyx_PyList_GetSlice(
            PyObject* src, Py_ssize_t start, Py_ssize_t stop) {
    PyObject *result;
    __Pyx_BEGIN_CRITICAL_SECTION(src);
    result = __Pyx_PyList_GetSlice_locked(src, start, stop);
    __Pyx_END_CRITICAL_SECTION();
    return result;
}
#endif // CYTHON_COMPILING_IN_CPYTHON

/* SwapException */
#if CYTHON_FAST_THREAD_STATE
static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
    PyObject *tmp_type, *tmp_value, *tmp_tb;
  #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4
    _PyErr_StackItem *exc_info = tstate->exc_info;
    tmp_value = exc_info->exc_value;
    exc_info->exc_value = *value;
    if (tmp_value == NULL || tmp_value == Py_None) {
        Py_XDECREF(tmp_value);
        tmp_value = NULL;
        tmp_type = NULL;
        tmp_tb = NULL;
    } else {
        tmp_type = (PyObject*) Py_TYPE(tmp_value);
        Py_INCREF(tmp_type);
        #if CYTHON_COMPILING_IN_CPYTHON
        tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback;
        Py_XINCREF(tmp_tb);
        #else
        tmp_tb = PyException_GetTraceback(tmp_value);
        #endif
    }
  #elif CYTHON_USE_EXC_INFO_STACK
    _PyErr_StackItem *exc_info = tstate->exc_info;
    tmp_type = exc_info->exc_type;
    tmp_value = exc_info->exc_value;
    tmp_tb = exc_info->exc_traceback;
    exc_info->exc_type = *type;
    exc_info->exc_value = *value;
    exc_info->exc_traceback = *tb;
  #else
    tmp_type = tstate->exc_type;
    tmp_value = tstate->exc_value;
    tmp_tb = tstate->exc_traceback;
    tstate->exc_type = *type;
    tstate->exc_value = *value;
    tstate->exc_traceback = *tb;
  #endif
    *type = tmp_type;
    *value = tmp_value;
    *tb = tmp_tb;
}
#else
static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) {
    PyObject *tmp_type, *tmp_value, *tmp_tb;
    PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb);
    PyErr_SetExcInfo(*type, *value, *tb);
    *type = tmp_type;
    *value = tmp_value;
    *tb = tmp_tb;
}
#endif

/* ArgTypeTestFunc (used by ArgTypeTest) */
static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact)
{
    __Pyx_TypeName type_name;
    __Pyx_TypeName obj_type_name;
    PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode;
    int from_annotation_subclass = 0;
    if (unlikely(!type)) {
        PyErr_SetString(PyExc_SystemError, "Missing type object");
        return 0;
    }
    else if (!exact) {
        if (likely(__Pyx_TypeCheck(obj, type))) return 1;
    } else if (exact == 2) {
        if (__Pyx_TypeCheck(obj, type)) {
            from_annotation_subclass = 1;
            extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately;
        }
    }
    type_name = __Pyx_PyType_GetFullyQualifiedName(type);
    obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj));
    PyErr_Format(PyExc_TypeError,
        "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME
        ", got " __Pyx_FMT_TYPENAME ")"
#if __PYX_LIMITED_VERSION_HEX < 0x030C0000
        "%s%U"
#endif
        , name, type_name, obj_type_name
#if __PYX_LIMITED_VERSION_HEX < 0x030C0000
        , (from_annotation_subclass ? ". " : ""), extra_info
#endif
        );
#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000
    if (exact == 2 && from_annotation_subclass) {
        PyObject *res;
        PyObject *vargs[2];
        vargs[0] = PyErr_GetRaisedException();
        vargs[1] = extra_info;
        res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL);
        Py_XDECREF(res);
        PyErr_SetRaisedException(vargs[0]);
    }
#endif
    __Pyx_DECREF_TypeName(type_name);
    __Pyx_DECREF_TypeName(obj_type_name);
    return 0;
}

/* RaiseTooManyValuesToUnpack */
static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) {
    PyErr_Format(PyExc_ValueError,
                 "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected);
}

/* RaiseNeedMoreValuesToUnpack */
static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) {
    PyErr_Format(PyExc_ValueError,
                 "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack",
                 index, (index == 1) ? "" : "s");
}

/* RaiseNoneIterError */
static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) {
    PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
}

/* GetAttr */
static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) {
#if CYTHON_USE_TYPE_SLOTS
    if (likely(PyUnicode_Check(n)))
        return __Pyx_PyObject_GetAttrStr(o, n);
#endif
    return PyObject_GetAttr(o, n);
}

/* PyObjectCall2Args (used by CallUnboundCMethod1) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) {
    PyObject *args[3] = {NULL, arg1, arg2};
    return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
}

/* CallUnboundCMethod1 */
#if CYTHON_COMPILING_IN_CPYTHON
static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg) {
    int was_initialized =  __Pyx_CachedCFunction_GetAndSetInitializing(cfunc);
    if (likely(was_initialized == 2 && cfunc->func)) {
        int flag = cfunc->flag;
        if (flag == METH_O) {
            return __Pyx_CallCFunction(cfunc, self, arg);
        } else if (flag == METH_FASTCALL) {
            return __Pyx_CallCFunctionFast(cfunc, self, &arg, 1);
        } else if (flag == (METH_FASTCALL | METH_KEYWORDS)) {
            return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, &arg, 1, NULL);
        }
    }
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    else if (unlikely(was_initialized == 1)) {
        __Pyx_CachedCFunction tmp_cfunc = {
#ifndef __cplusplus
            0
#endif
        };
        tmp_cfunc.type = cfunc->type;
        tmp_cfunc.method_name = cfunc->method_name;
        return __Pyx__CallUnboundCMethod1(&tmp_cfunc, self, arg);
    }
#endif
    PyObject* result = __Pyx__CallUnboundCMethod1(cfunc, self, arg);
    __Pyx_CachedCFunction_SetFinishedInitializing(cfunc);
    return result;
}
#endif
static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg){
    PyObject *result = NULL;
    if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL;
#if CYTHON_COMPILING_IN_CPYTHON
    if (cfunc->func && (cfunc->flag & METH_VARARGS)) {
        PyObject *args = PyTuple_New(1);
        if (unlikely(!args)) return NULL;
        Py_INCREF(arg);
        PyTuple_SET_ITEM(args, 0, arg);
        if (cfunc->flag & METH_KEYWORDS)
            result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL);
        else
            result = __Pyx_CallCFunction(cfunc, self, args);
        Py_DECREF(args);
    } else
#endif
    {
        result = __Pyx_PyObject_Call2Args(cfunc->method, self, arg);
    }
    return result;
}

/* dict_getitem_default */
static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) {
    PyObject* value;
#if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000
    value = PyDict_GetItemWithError(d, key);
    if (unlikely(!value)) {
        if (unlikely(PyErr_Occurred()))
            return NULL;
        value = default_value;
    }
    Py_INCREF(value);
    if ((1));
#else
    if (PyBytes_CheckExact(key) || PyUnicode_CheckExact(key) || PyLong_CheckExact(key)) {
        value = PyDict_GetItem(d, key);
        if (unlikely(!value)) {
            value = default_value;
        }
        Py_INCREF(value);
    }
#endif
    else {
        if (default_value == Py_None)
            value = __Pyx_CallUnboundCMethod1(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_get, d, key);
        else
            value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_get, d, key, default_value);
    }
    return value;
}

/* AllocateExtensionType */
static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) {
    if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) {
        allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc);
        return alloc_func(t, 0);
    } else {
        newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc);
    #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
        if (!tp_new) {
            PyObject *new_str = PyUnicode_FromString("__new__");
            if (likely(new_str)) {
                PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL);
                Py_DECREF(new_str);
                return o;
            } else
                return NULL;
        } else
    #endif
        return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0);
    }
}

/* CallTypeTraverse */
#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000)
#else
static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) {
    #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000
    if (__Pyx_get_runtime_version() < 0x03090000) return 0;
    #endif
    if (!always_call) {
        PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*);
        unsigned long flags = PyType_GetFlags(base);
        if (flags & Py_TPFLAGS_HEAPTYPE) {
            return 0;
        }
    }
    Py_VISIT((PyObject*)Py_TYPE(o));
    return 0;
}
#endif

/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */
#if CYTHON_COMPILING_IN_LIMITED_API
static Py_ssize_t __Pyx_GetTypeDictOffset(void) {
    PyObject *tp_dictoffset_o;
    Py_ssize_t tp_dictoffset;
    tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__");
    if (unlikely(!tp_dictoffset_o)) return -1;
    tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o);
    Py_DECREF(tp_dictoffset_o);
    if (unlikely(tp_dictoffset == 0)) {
        PyErr_SetString(
            PyExc_TypeError,
            "'type' doesn't have a dictoffset");
        return -1;
    } else if (unlikely(tp_dictoffset < 0)) {
        PyErr_SetString(
            PyExc_TypeError,
            "'type' has an unexpected negative dictoffset. "
            "Please report this as Cython bug");
        return -1;
    }
    return tp_dictoffset;
}
static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) {
    static Py_ssize_t tp_dictoffset = 0;
    if (unlikely(tp_dictoffset == 0)) {
        tp_dictoffset = __Pyx_GetTypeDictOffset();
        if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) {
            tp_dictoffset = 0; // try again next time?
            return NULL;
        }
    }
    return *(PyObject**)((char*)tp + tp_dictoffset);
}
#endif

/* SetItemOnTypeDict (used by FixUpExtensionType) */
static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) {
    int result;
    PyObject *tp_dict;
#if CYTHON_COMPILING_IN_LIMITED_API
    tp_dict = __Pyx_GetTypeDict(tp);
    if (unlikely(!tp_dict)) return -1;
#else
    tp_dict = tp->tp_dict;
#endif
    result = PyDict_SetItem(tp_dict, k, v);
    if (likely(!result)) {
        PyType_Modified(tp);
        if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) {
            PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name,  (PyObject *) tp, k, NULL);
            if (!setNameResult) return -1;
            Py_DECREF(setNameResult);
        }
    }
    return result;
}

/* FixUpExtensionType */
static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) {
#if __PYX_LIMITED_VERSION_HEX > 0x030900B1
    CYTHON_UNUSED_VAR(spec);
    CYTHON_UNUSED_VAR(type);
    CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict);
#else
    const PyType_Slot *slot = spec->slots;
    int changed = 0;
#if !CYTHON_COMPILING_IN_LIMITED_API
    while (slot && slot->slot && slot->slot != Py_tp_members)
        slot++;
    if (slot && slot->slot == Py_tp_members) {
#if !CYTHON_COMPILING_IN_CPYTHON
        const
#endif  // !CYTHON_COMPILING_IN_CPYTHON)
            PyMemberDef *memb = (PyMemberDef*) slot->pfunc;
        while (memb && memb->name) {
            if (memb->name[0] == '_' && memb->name[1] == '_') {
                if (strcmp(memb->name, "__weaklistoffset__") == 0) {
                    assert(memb->type == T_PYSSIZET);
                    assert(memb->flags == READONLY);
                    type->tp_weaklistoffset = memb->offset;
                    changed = 1;
                }
                else if (strcmp(memb->name, "__dictoffset__") == 0) {
                    assert(memb->type == T_PYSSIZET);
                    assert(memb->flags == READONLY);
                    type->tp_dictoffset = memb->offset;
                    changed = 1;
                }
#if CYTHON_METH_FASTCALL
                else if (strcmp(memb->name, "__vectorcalloffset__") == 0) {
                    assert(memb->type == T_PYSSIZET);
                    assert(memb->flags == READONLY);
                    type->tp_vectorcall_offset = memb->offset;
                    changed = 1;
                }
#endif  // CYTHON_METH_FASTCALL
#if !CYTHON_COMPILING_IN_PYPY
                else if (strcmp(memb->name, "__module__") == 0) {
                    PyObject *descr;
                    assert(memb->type == T_OBJECT);
                    assert(memb->flags == 0 || memb->flags == READONLY);
                    descr = PyDescr_NewMember(type, memb);
                    if (unlikely(!descr))
                        return -1;
                    int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr);
                    Py_DECREF(descr);
                    if (unlikely(set_item_result < 0)) {
                        return -1;
                    }
                    changed = 1;
                }
#endif  // !CYTHON_COMPILING_IN_PYPY
            }
            memb++;
        }
    }
#endif  // !CYTHON_COMPILING_IN_LIMITED_API
#if !CYTHON_COMPILING_IN_PYPY
    slot = spec->slots;
    while (slot && slot->slot && slot->slot != Py_tp_getset)
        slot++;
    if (slot && slot->slot == Py_tp_getset) {
        PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc;
        while (getset && getset->name) {
            if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) {
                PyObject *descr = PyDescr_NewGetSet(type, getset);
                if (unlikely(!descr))
                    return -1;
                #if CYTHON_COMPILING_IN_LIMITED_API
                PyObject *pyname = PyUnicode_FromString(getset->name);
                if (unlikely(!pyname)) {
                    Py_DECREF(descr);
                    return -1;
                }
                int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr);
                Py_DECREF(pyname);
                #else
                CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict);
                int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr);
                #endif
                Py_DECREF(descr);
                if (unlikely(set_item_result < 0)) {
                    return -1;
                }
                changed = 1;
            }
            ++getset;
        }
    }
#else
    CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict);
#endif  // !CYTHON_COMPILING_IN_PYPY
    if (changed)
        PyType_Modified(type);
#endif  // PY_VERSION_HEX > 0x030900B1
    return 0;
}

/* PyObjectCallNoArg (used by PyObjectCallMethod0) */
static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) {
    PyObject *arg[2] = {NULL, NULL};
    return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
}

/* PyObjectGetMethod (used by PyObjectCallMethod0) */
#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)))
static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) {
    PyObject *attr;
#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP
    __Pyx_TypeName type_name;
    PyTypeObject *tp = Py_TYPE(obj);
    PyObject *descr;
    descrgetfunc f = NULL;
    PyObject **dictptr, *dict;
    int meth_found = 0;
    assert (*method == NULL);
    if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) {
        attr = __Pyx_PyObject_GetAttrStr(obj, name);
        goto try_unpack;
    }
    if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) {
        return 0;
    }
    descr = _PyType_Lookup(tp, name);
    if (likely(descr != NULL)) {
        Py_INCREF(descr);
#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR
        if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR))
#else
        #ifdef __Pyx_CyFunction_USED
        if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr)))
        #else
        if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type)))
        #endif
#endif
        {
            meth_found = 1;
        } else {
            f = Py_TYPE(descr)->tp_descr_get;
            if (f != NULL && PyDescr_IsData(descr)) {
                attr = f(descr, obj, (PyObject *)Py_TYPE(obj));
                Py_DECREF(descr);
                goto try_unpack;
            }
        }
    }
    dictptr = _PyObject_GetDictPtr(obj);
    if (dictptr != NULL && (dict = *dictptr) != NULL) {
        Py_INCREF(dict);
        attr = __Pyx_PyDict_GetItemStr(dict, name);
        if (attr != NULL) {
            Py_INCREF(attr);
            Py_DECREF(dict);
            Py_XDECREF(descr);
            goto try_unpack;
        }
        Py_DECREF(dict);
    }
    if (meth_found) {
        *method = descr;
        return 1;
    }
    if (f != NULL) {
        attr = f(descr, obj, (PyObject *)Py_TYPE(obj));
        Py_DECREF(descr);
        goto try_unpack;
    }
    if (likely(descr != NULL)) {
        *method = descr;
        return 0;
    }
    type_name = __Pyx_PyType_GetFullyQualifiedName(tp);
    PyErr_Format(PyExc_AttributeError,
                 "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'",
                 type_name, name);
    __Pyx_DECREF_TypeName(type_name);
    return 0;
#else
    attr = __Pyx_PyObject_GetAttrStr(obj, name);
    goto try_unpack;
#endif
try_unpack:
#if CYTHON_UNPACK_METHODS
    if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) {
        PyObject *function = PyMethod_GET_FUNCTION(attr);
        Py_INCREF(function);
        Py_DECREF(attr);
        *method = function;
        return 1;
    }
#endif
    *method = attr;
    return 0;
}
#endif

/* PyObjectCallMethod0 (used by PyType_Ready) */
static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) {
#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))
    PyObject *args[1] = {obj};
    (void) __Pyx_PyObject_CallOneArg;
    (void) __Pyx_PyObject_CallNoArg;
    return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
#else
    PyObject *method = NULL, *result = NULL;
    int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method);
    if (likely(is_method)) {
        result = __Pyx_PyObject_CallOneArg(method, obj);
        Py_DECREF(method);
        return result;
    }
    if (unlikely(!method)) goto bad;
    result = __Pyx_PyObject_CallNoArg(method);
    Py_DECREF(method);
bad:
    return result;
#endif
}

/* ValidateBasesTuple (used by PyType_Ready) */
#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS
static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) {
    Py_ssize_t i, n;
#if CYTHON_ASSUME_SAFE_SIZE
    n = PyTuple_GET_SIZE(bases);
#else
    n = PyTuple_Size(bases);
    if (unlikely(n < 0)) return -1;
#endif
    for (i = 1; i < n; i++)
    {
        PyTypeObject *b;
#if CYTHON_AVOID_BORROWED_REFS
        PyObject *b0 = PySequence_GetItem(bases, i);
        if (!b0) return -1;
#elif CYTHON_ASSUME_SAFE_MACROS
        PyObject *b0 = PyTuple_GET_ITEM(bases, i);
#else
        PyObject *b0 = PyTuple_GetItem(bases, i);
        if (!b0) return -1;
#endif
        b = (PyTypeObject*) b0;
        if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE))
        {
            __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b);
            PyErr_Format(PyExc_TypeError,
                "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name);
            __Pyx_DECREF_TypeName(b_name);
#if CYTHON_AVOID_BORROWED_REFS
            Py_DECREF(b0);
#endif
            return -1;
        }
        if (dictoffset == 0)
        {
            Py_ssize_t b_dictoffset = 0;
#if CYTHON_USE_TYPE_SLOTS
            b_dictoffset = b->tp_dictoffset;
#else
            PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__");
            if (!py_b_dictoffset) goto dictoffset_return;
            b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset);
            Py_DECREF(py_b_dictoffset);
            if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return;
#endif
            if (b_dictoffset) {
                {
                    __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b);
                    PyErr_Format(PyExc_TypeError,
                        "extension type '%.200s' has no __dict__ slot, "
                        "but base type '" __Pyx_FMT_TYPENAME "' has: "
                        "either add 'cdef dict __dict__' to the extension type "
                        "or add '__slots__ = [...]' to the base type",
                        type_name, b_name);
                    __Pyx_DECREF_TypeName(b_name);
                }
#if !CYTHON_USE_TYPE_SLOTS
              dictoffset_return:
#endif
#if CYTHON_AVOID_BORROWED_REFS
                Py_DECREF(b0);
#endif
                return -1;
            }
        }
#if CYTHON_AVOID_BORROWED_REFS
        Py_DECREF(b0);
#endif
    }
    return 0;
}
#endif

/* PyType_Ready */
CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) {
    while (t) {
        PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*);
        if (bases) {
            return 1;
        }
        t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*);
    }
    return 0;
}
static int __Pyx_PyType_Ready(PyTypeObject *t) {
#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION)
    (void)__Pyx_PyObject_CallMethod0;
#if CYTHON_USE_TYPE_SPECS
    (void)__Pyx_validate_bases_tuple;
#endif
    return PyType_Ready(t);
#else
    int r;
    if (!__Pyx_PyType_HasMultipleInheritance(t)) {
        return PyType_Ready(t);
    }
    PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*);
    if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1))
        return -1;
#if !defined(PYSTON_MAJOR_VERSION)
    {
        int gc_was_enabled;
    #if PY_VERSION_HEX >= 0x030A00b1
        gc_was_enabled = PyGC_Disable();
        (void)__Pyx_PyObject_CallMethod0;
    #else
        PyObject *ret, *py_status;
        PyObject *gc = NULL;
        #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\
                !CYTHON_COMPILING_IN_GRAAL
        gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc);
        #endif
        if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc);
        if (unlikely(!gc)) return -1;
        py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled);
        if (unlikely(!py_status)) {
            Py_DECREF(gc);
            return -1;
        }
        gc_was_enabled = __Pyx_PyObject_IsTrue(py_status);
        Py_DECREF(py_status);
        if (gc_was_enabled > 0) {
            ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable);
            if (unlikely(!ret)) {
                Py_DECREF(gc);
                return -1;
            }
            Py_DECREF(ret);
        } else if (unlikely(gc_was_enabled == -1)) {
            Py_DECREF(gc);
            return -1;
        }
    #endif
        t->tp_flags |= Py_TPFLAGS_HEAPTYPE;
#if PY_VERSION_HEX >= 0x030A0000
        t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
#endif
#else
        (void)__Pyx_PyObject_CallMethod0;
#endif
    r = PyType_Ready(t);
#if !defined(PYSTON_MAJOR_VERSION)
        t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE;
    #if PY_VERSION_HEX >= 0x030A00b1
        if (gc_was_enabled)
            PyGC_Enable();
    #else
        if (gc_was_enabled) {
            PyObject *tp, *v, *tb;
            PyErr_Fetch(&tp, &v, &tb);
            ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable);
            if (likely(ret || r == -1)) {
                Py_XDECREF(ret);
                PyErr_Restore(tp, v, tb);
            } else {
                Py_XDECREF(tp);
                Py_XDECREF(v);
                Py_XDECREF(tb);
                r = -1;
            }
        }
        Py_DECREF(gc);
    #endif
    }
#endif
    return r;
#endif
}

/* SetVTable */
static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) {
    PyObject *ob = PyCapsule_New(vtable, 0, 0);
    if (unlikely(!ob))
        goto bad;
#if CYTHON_COMPILING_IN_LIMITED_API
    if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0))
#else
    if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0))
#endif
        goto bad;
    Py_DECREF(ob);
    return 0;
bad:
    Py_XDECREF(ob);
    return -1;
}

/* GetVTable (used by MergeVTables) */
static void* __Pyx_GetVtable(PyTypeObject *type) {
    void* ptr;
#if CYTHON_COMPILING_IN_LIMITED_API
    PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable);
#else
    PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable);
#endif
    if (!ob)
        goto bad;
    ptr = PyCapsule_GetPointer(ob, 0);
    if (!ptr && !PyErr_Occurred())
        PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type");
    Py_DECREF(ob);
    return ptr;
bad:
    Py_XDECREF(ob);
    return NULL;
}

/* MergeVTables */
static int __Pyx_MergeVtables(PyTypeObject *type) {
    int i=0;
    Py_ssize_t size;
    void** base_vtables;
    __Pyx_TypeName tp_base_name = NULL;
    __Pyx_TypeName base_name = NULL;
    void* unknown = (void*)-1;
    PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*);
    int base_depth = 0;
    {
        PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*);
        while (base) {
            base_depth += 1;
            base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*);
        }
    }
    base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1));
    base_vtables[0] = unknown;
#if CYTHON_COMPILING_IN_LIMITED_API
    size = PyTuple_Size(bases);
    if (size < 0) goto other_failure;
#else
    size = PyTuple_GET_SIZE(bases);
#endif
    for (i = 1; i < size; i++) {
        PyObject *basei;
        void* base_vtable;
#if CYTHON_AVOID_BORROWED_REFS
        basei = PySequence_GetItem(bases, i);
        if (unlikely(!basei)) goto other_failure;
#elif !CYTHON_ASSUME_SAFE_MACROS
        basei = PyTuple_GetItem(bases, i);
        if (unlikely(!basei)) goto other_failure;
#else
        basei = PyTuple_GET_ITEM(bases, i);
#endif
        base_vtable = __Pyx_GetVtable((PyTypeObject*)basei);
#if CYTHON_AVOID_BORROWED_REFS
        Py_DECREF(basei);
#endif
        if (base_vtable != NULL) {
            int j;
            PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*);
            for (j = 0; j < base_depth; j++) {
                if (base_vtables[j] == unknown) {
                    base_vtables[j] = __Pyx_GetVtable(base);
                    base_vtables[j + 1] = unknown;
                }
                if (base_vtables[j] == base_vtable) {
                    break;
                } else if (base_vtables[j] == NULL) {
                    goto bad;
                }
                base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*);
            }
        }
    }
    PyErr_Clear();
    PyMem_Free(base_vtables);
    return 0;
bad:
    {
        PyTypeObject* basei = NULL;
        PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*);
        tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base);
#if CYTHON_AVOID_BORROWED_REFS
        basei = (PyTypeObject*)PySequence_GetItem(bases, i);
        if (unlikely(!basei)) goto really_bad;
#elif !CYTHON_ASSUME_SAFE_MACROS
        basei = (PyTypeObject*)PyTuple_GetItem(bases, i);
        if (unlikely(!basei)) goto really_bad;
#else
        basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i);
#endif
        base_name = __Pyx_PyType_GetFullyQualifiedName(basei);
#if CYTHON_AVOID_BORROWED_REFS
        Py_DECREF(basei);
#endif
    }
    PyErr_Format(PyExc_TypeError,
        "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name);
#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS
really_bad: // bad has failed!
#endif
    __Pyx_DECREF_TypeName(tp_base_name);
    __Pyx_DECREF_TypeName(base_name);
#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS
other_failure:
#endif
    PyMem_Free(base_vtables);
    return -1;
}

/* DelItemOnTypeDict (used by SetupReduce) */
static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k) {
    int result;
    PyObject *tp_dict;
#if CYTHON_COMPILING_IN_LIMITED_API
    tp_dict = __Pyx_GetTypeDict(tp);
    if (unlikely(!tp_dict)) return -1;
#else
    tp_dict = tp->tp_dict;
#endif
    result = PyDict_DelItem(tp_dict, k);
    if (likely(!result)) PyType_Modified(tp);
    return result;
}

/* SetupReduce */
static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) {
  int ret;
  PyObject *name_attr;
  name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, __pyx_mstate_global->__pyx_n_u_name_2);
  if (likely(name_attr)) {
      ret = PyObject_RichCompareBool(name_attr, name, Py_EQ);
  } else {
      ret = -1;
  }
  if (unlikely(ret < 0)) {
      PyErr_Clear();
      ret = 0;
  }
  Py_XDECREF(name_attr);
  return ret;
}
static int __Pyx_setup_reduce(PyObject* type_obj) {
    int ret = 0;
    PyObject *object_reduce = NULL;
    PyObject *object_getstate = NULL;
    PyObject *object_reduce_ex = NULL;
    PyObject *reduce = NULL;
    PyObject *reduce_ex = NULL;
    PyObject *reduce_cython = NULL;
    PyObject *setstate = NULL;
    PyObject *setstate_cython = NULL;
    PyObject *getstate = NULL;
#if CYTHON_USE_PYTYPE_LOOKUP
    getstate = _PyType_Lookup((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_getstate);
#else
    getstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_getstate);
    if (!getstate && PyErr_Occurred()) {
        goto __PYX_BAD;
    }
#endif
    if (getstate) {
#if CYTHON_USE_PYTYPE_LOOKUP
        object_getstate = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate);
#else
        object_getstate = __Pyx_PyObject_GetAttrStrNoError((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate);
        if (!object_getstate && PyErr_Occurred()) {
            goto __PYX_BAD;
        }
#endif
        if (object_getstate != getstate) {
            goto __PYX_GOOD;
        }
    }
#if CYTHON_USE_PYTYPE_LOOKUP
    object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD;
#else
    object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD;
#endif
    reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (unlikely(!reduce_ex)) goto __PYX_BAD;
    if (reduce_ex == object_reduce_ex) {
#if CYTHON_USE_PYTYPE_LOOKUP
        object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD;
#else
        object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD;
#endif
        reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce); if (unlikely(!reduce)) goto __PYX_BAD;
        if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_mstate_global->__pyx_n_u_reduce_cython)) {
            reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython);
            if (likely(reduce_cython)) {
                ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce, reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD;
                ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD;
            } else if (reduce == object_reduce || PyErr_Occurred()) {
                goto __PYX_BAD;
            }
            setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate);
            if (!setstate) PyErr_Clear();
            if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_mstate_global->__pyx_n_u_setstate_cython)) {
                setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython);
                if (likely(setstate_cython)) {
                    ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate, setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD;
                    ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD;
                } else if (!setstate || PyErr_Occurred()) {
                    goto __PYX_BAD;
                }
            }
            PyType_Modified((PyTypeObject*)type_obj);
        }
    }
    goto __PYX_GOOD;
__PYX_BAD:
    if (!PyErr_Occurred()) {
        __Pyx_TypeName type_obj_name =
            __Pyx_PyType_GetFullyQualifiedName((PyTypeObject*)type_obj);
        PyErr_Format(PyExc_RuntimeError,
            "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name);
        __Pyx_DECREF_TypeName(type_obj_name);
    }
    ret = -1;
__PYX_GOOD:
#if !CYTHON_USE_PYTYPE_LOOKUP
    Py_XDECREF(object_reduce);
    Py_XDECREF(object_reduce_ex);
    Py_XDECREF(object_getstate);
    Py_XDECREF(getstate);
#endif
    Py_XDECREF(reduce);
    Py_XDECREF(reduce_ex);
    Py_XDECREF(reduce_cython);
    Py_XDECREF(setstate);
    Py_XDECREF(setstate_cython);
    return ret;
}

/* TypeImport */
#ifndef __PYX_HAVE_RT_ImportType_3_2_5
#define __PYX_HAVE_RT_ImportType_3_2_5
static PyTypeObject *__Pyx_ImportType_3_2_5(PyObject *module, const char *module_name, const char *class_name,
    size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_5 check_size)
{
    PyObject *result = 0;
    Py_ssize_t basicsize;
    Py_ssize_t itemsize;
#if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API)
    PyObject *py_basicsize;
    PyObject *py_itemsize;
#endif
    result = PyObject_GetAttrString(module, class_name);
    if (!result)
        goto bad;
    if (!PyType_Check(result)) {
        PyErr_Format(PyExc_TypeError,
            "%.200s.%.200s is not a type object",
            module_name, class_name);
        goto bad;
    }
#if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) )
    basicsize = ((PyTypeObject *)result)->tp_basicsize;
    itemsize = ((PyTypeObject *)result)->tp_itemsize;
#else
    if (size == 0) {
        return (PyTypeObject *)result;
    }
    py_basicsize = PyObject_GetAttrString(result, "__basicsize__");
    if (!py_basicsize)
        goto bad;
    basicsize = PyLong_AsSsize_t(py_basicsize);
    Py_DECREF(py_basicsize);
    py_basicsize = 0;
    if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred())
        goto bad;
    py_itemsize = PyObject_GetAttrString(result, "__itemsize__");
    if (!py_itemsize)
        goto bad;
    itemsize = PyLong_AsSsize_t(py_itemsize);
    Py_DECREF(py_itemsize);
    py_itemsize = 0;
    if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred())
        goto bad;
#endif
    if (itemsize) {
        if (size % alignment) {
            alignment = size % alignment;
        }
        if (itemsize < (Py_ssize_t)alignment)
            itemsize = (Py_ssize_t)alignment;
    }
    if ((size_t)(basicsize + itemsize) < size) {
        PyErr_Format(PyExc_ValueError,
            "%.200s.%.200s size changed, may indicate binary incompatibility. "
            "Expected %zd from C header, got %zd from PyObject",
            module_name, class_name, size, basicsize+itemsize);
        goto bad;
    }
    if (check_size == __Pyx_ImportType_CheckSize_Error_3_2_5 &&
            ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) {
        PyErr_Format(PyExc_ValueError,
            "%.200s.%.200s size changed, may indicate binary incompatibility. "
            "Expected %zd from C header, got %zd-%zd from PyObject",
            module_name, class_name, size, basicsize, basicsize+itemsize);
        goto bad;
    }
    else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_2_5 && (size_t)basicsize > size) {
        if (PyErr_WarnFormat(NULL, 0,
                "%.200s.%.200s size changed, may indicate binary incompatibility. "
                "Expected %zd from C header, got %zd from PyObject",
                module_name, class_name, size, basicsize) < 0) {
            goto bad;
        }
    }
    return (PyTypeObject *)result;
bad:
    Py_XDECREF(result);
    return NULL;
}
#endif

/* PxdImportShared (used by VoidPtrImport) */
#ifndef __PYX_HAVE_RT_ImportFromPxd_3_2_5
#define __PYX_HAVE_RT_ImportFromPxd_3_2_5
static int __Pyx_ImportFromPxd_3_2_5(PyObject *module, const char *name, void **p, const char *sig, const char *what) {
    PyObject *d = 0;
    PyObject *cobj = 0;
    d = PyObject_GetAttrString(module, "__pyx_capi__");
    if (!d)
        goto bad;
#if (defined(Py_LIMITED_API) && Py_LIMITED_API >= 0x030d0000) || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030d0000)
    PyDict_GetItemStringRef(d, name, &cobj);
#else
    cobj = PyDict_GetItemString(d, name);
    Py_XINCREF(cobj);
#endif
    if (!cobj) {
        PyErr_Format(PyExc_ImportError,
            "%.200s does not export expected C %.8s %.200s",
                PyModule_GetName(module), what, name);
        goto bad;
    }
    if (!PyCapsule_IsValid(cobj, sig)) {
        PyErr_Format(PyExc_TypeError,
            "C %.8s %.200s.%.200s has wrong signature (expected %.500s, got %.500s)",
             what, PyModule_GetName(module), name, sig, PyCapsule_GetName(cobj));
        goto bad;
    }
    *p = PyCapsule_GetPointer(cobj, sig);
    if (!(*p))
        goto bad;
    Py_DECREF(d);
    Py_DECREF(cobj);
    return 0;
bad:
    Py_XDECREF(d);
    Py_XDECREF(cobj);
    return -1;
}
#endif

/* VoidPtrImport */
#ifndef __PYX_HAVE_RT_ImportVoidPtr_3_2_5
#define __PYX_HAVE_RT_ImportVoidPtr_3_2_5
static int __Pyx_ImportVoidPtr_3_2_5(PyObject *module, const char *name, void **p, const char *sig) {
    return __Pyx_ImportFromPxd_3_2_5(module, name, p, sig, "variable");
}
#endif

/* FunctionImport */
#ifndef __PYX_HAVE_RT_ImportFunction_3_2_5
#define __PYX_HAVE_RT_ImportFunction_3_2_5
static int __Pyx_ImportFunction_3_2_5(PyObject *module, const char *funcname, void (**f)(void), const char *sig) {
    union {
        void (*fp)(void);
        void *p;
    } tmp;
    int result = __Pyx_ImportFromPxd_3_2_5(module, funcname, &tmp.p, sig, "function");
    if (result == 0) {
        *f = tmp.fp;
    }
    return result;
}
#endif

/* dict_setdefault (used by CLineInTraceback) */
static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) {
    PyObject* value;
#if __PYX_LIMITED_VERSION_HEX >= 0x030F0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4)
    PyDict_SetDefaultRef(d, key, default_value, &value);
#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000
    PyObject *args[] = {d, key, default_value};
    value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
#elif CYTHON_COMPILING_IN_LIMITED_API
    value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL);
#else
    value = PyDict_SetDefault(d, key, default_value);
    if (unlikely(!value)) return NULL;
    Py_INCREF(value);
#endif
    return value;
}

/* CLineInTraceback (used by AddTraceback) */
#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME
#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o))
#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL);
#else
PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) {
    PyObject **dict_ptr = _PyObject_GetDictPtr(o);
    return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL;
}
#endif
static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) {
    PyObject *use_cline = NULL;
    PyObject *ptype, *pvalue, *ptraceback;
    PyObject *cython_runtime_dict;
    CYTHON_MAYBE_UNUSED_VAR(tstate);
    if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) {
        return c_line;
    }
    __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback);
    cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime);
    if (likely(cython_runtime_dict)) {
        __PYX_PY_DICT_LOOKUP_IF_MODIFIED(
            use_cline, cython_runtime_dict,
            __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False))
    }
    if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) {
        c_line = 0;
    }
    Py_XDECREF(use_cline);
    Py_XDECREF(cython_runtime_dict);
    __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback);
    return c_line;
}
#endif

/* CodeObjectCache (used by AddTraceback) */
static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) {
    int start = 0, mid = 0, end = count - 1;
    if (end >= 0 && code_line > entries[end].code_line) {
        return count;
    }
    while (start < end) {
        mid = start + (end - start) / 2;
        if (code_line < entries[mid].code_line) {
            end = mid;
        } else if (code_line > entries[mid].code_line) {
             start = mid + 1;
        } else {
            return mid;
        }
    }
    if (code_line <= entries[mid].code_line) {
        return mid;
    } else {
        return mid + 1;
    }
}
static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) {
    __Pyx_CachedCodeObjectType* code_object;
    int pos;
    if (unlikely(!code_line) || unlikely(!code_cache->entries)) {
        return NULL;
    }
    pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line);
    if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) {
        return NULL;
    }
    code_object = code_cache->entries[pos].code_object;
    Py_INCREF(code_object);
    return code_object;
}
static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) {
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS
    (void)__pyx__find_code_object;
    return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss.
#else
    struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache;
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count);
    if (old_count < 0) {
        __pyx_atomic_decr_acq_rel(&code_cache->accessor_count);
        return NULL;
    }
#endif
    __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line);
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    __pyx_atomic_decr_acq_rel(&code_cache->accessor_count);
#endif
    return result;
#endif
}
static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object)
{
    int pos, i;
    __Pyx_CodeObjectCacheEntry* entries = code_cache->entries;
    if (unlikely(!code_line)) {
        return;
    }
    if (unlikely(!entries)) {
        entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry));
        if (likely(entries)) {
            code_cache->entries = entries;
            code_cache->max_count = 64;
            code_cache->count = 1;
            entries[0].code_line = code_line;
            entries[0].code_object = code_object;
            Py_INCREF(code_object);
        }
        return;
    }
    pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line);
    if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) {
        __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object;
        entries[pos].code_object = code_object;
        Py_INCREF(code_object);
        Py_DECREF(tmp);
        return;
    }
    if (code_cache->count == code_cache->max_count) {
        int new_max = code_cache->max_count + 64;
        entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc(
            code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry));
        if (unlikely(!entries)) {
            return;
        }
        code_cache->entries = entries;
        code_cache->max_count = new_max;
    }
    for (i=code_cache->count; i>pos; i--) {
        entries[i] = entries[i-1];
    }
    entries[pos].code_line = code_line;
    entries[pos].code_object = code_object;
    code_cache->count++;
    Py_INCREF(code_object);
}
static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) {
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS
    (void)__pyx__insert_code_object;
    return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail.
#else
    struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache;
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    __pyx_nonatomic_int_type expected = 0;
    if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) {
        return;
    }
#endif
    __pyx__insert_code_object(code_cache, code_line, code_object);
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
    __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN);
#endif
#endif
}

/* AddTraceback */
#include "compile.h"
#include "frameobject.h"
#include "traceback.h"
#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION)
  #ifndef Py_BUILD_CORE
    #define Py_BUILD_CORE 1
  #endif
  #include "internal/pycore_frame.h"
#endif
#if CYTHON_COMPILING_IN_LIMITED_API
static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict,
                                                       PyObject *firstlineno, PyObject *name) {
    PyObject *replace = NULL;
    if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL;
    if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL;
    replace = PyObject_GetAttrString(code, "replace");
    if (likely(replace)) {
        PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict);
        Py_DECREF(replace);
        return result;
    }
    PyErr_Clear();
    return NULL;
}
static void __Pyx_AddTraceback(const char *funcname, int c_line,
                               int py_line, const char *filename) {
    PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL;
    PyObject *replace = NULL, *getframe = NULL, *frame = NULL;
    PyObject *exc_type, *exc_value, *exc_traceback;
    int success = 0;
    if (c_line) {
        c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line);
    }
    PyErr_Fetch(&exc_type, &exc_value, &exc_traceback);
    code_object = __pyx_find_code_object(c_line ? -c_line : py_line);
    if (!code_object) {
        code_object = Py_CompileString("_getframe()", filename, Py_eval_input);
        if (unlikely(!code_object)) goto bad;
        py_py_line = PyLong_FromLong(py_line);
        if (unlikely(!py_py_line)) goto bad;
        if (c_line) {
            py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line);
        } else {
            py_funcname = PyUnicode_FromString(funcname);
        }
        if (unlikely(!py_funcname)) goto bad;
        dict = PyDict_New();
        if (unlikely(!dict)) goto bad;
        {
            PyObject *old_code_object = code_object;
            code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname);
            Py_DECREF(old_code_object);
        }
        if (unlikely(!code_object)) goto bad;
        __pyx_insert_code_object(c_line ? -c_line : py_line, code_object);
    } else {
        dict = PyDict_New();
    }
    getframe = PySys_GetObject("_getframe");
    if (unlikely(!getframe)) goto bad;
    if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad;
    frame = PyEval_EvalCode(code_object, dict, dict);
    if (unlikely(!frame) || frame == Py_None) goto bad;
    success = 1;
  bad:
    PyErr_Restore(exc_type, exc_value, exc_traceback);
    Py_XDECREF(code_object);
    Py_XDECREF(py_py_line);
    Py_XDECREF(py_funcname);
    Py_XDECREF(dict);
    Py_XDECREF(replace);
    if (success) {
        PyTraceBack_Here(
            (struct _frame*)frame);
    }
    Py_XDECREF(frame);
}
#else
static PyCodeObject* __Pyx_CreateCodeObjectForTraceback(
            const char *funcname, int c_line,
            int py_line, const char *filename) {
    PyCodeObject *py_code = NULL;
    PyObject *py_funcname = NULL;
    if (c_line) {
        py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line);
        if (!py_funcname) goto bad;
        funcname = PyUnicode_AsUTF8(py_funcname);
        if (!funcname) goto bad;
    }
    py_code = PyCode_NewEmpty(filename, funcname, py_line);
    Py_XDECREF(py_funcname);
    return py_code;
bad:
    Py_XDECREF(py_funcname);
    return NULL;
}
static void __Pyx_AddTraceback(const char *funcname, int c_line,
                               int py_line, const char *filename) {
    PyCodeObject *py_code = 0;
    PyFrameObject *py_frame = 0;
    PyThreadState *tstate = __Pyx_PyThreadState_Current;
    PyObject *ptype, *pvalue, *ptraceback;
    if (c_line) {
        c_line = __Pyx_CLineForTraceback(tstate, c_line);
    }
    py_code = __pyx_find_code_object(c_line ? -c_line : py_line);
    if (!py_code) {
        __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback);
        py_code = __Pyx_CreateCodeObjectForTraceback(
            funcname, c_line, py_line, filename);
        if (!py_code) {
            /* If the code object creation fails, then we should clear the
               fetched exception references and propagate the new exception */
            Py_XDECREF(ptype);
            Py_XDECREF(pvalue);
            Py_XDECREF(ptraceback);
            goto bad;
        }
        __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback);
        __pyx_insert_code_object(c_line ? -c_line : py_line, py_code);
    }
    py_frame = PyFrame_New(
        tstate,            /*PyThreadState *tstate,*/
        py_code,           /*PyCodeObject *code,*/
        __pyx_mstate_global->__pyx_d,    /*PyObject *globals,*/
        0                  /*PyObject *locals*/
    );
    if (!py_frame) goto bad;
    __Pyx_PyFrame_SetLineNumber(py_frame, py_line);
    PyTraceBack_Here(py_frame);
bad:
    Py_XDECREF(py_code);
    Py_XDECREF(py_frame);
}
#endif

/* CheckUnpickleChecksum */
static void __Pyx_RaiseUnpickleChecksumError(long checksum, long checksum1, long checksum2, long checksum3, const char *members) {
    PyObject *pickle_module = PyImport_ImportModule("pickle");
    if (unlikely(!pickle_module)) return;
    PyObject *pickle_error = PyObject_GetAttrString(pickle_module, "PickleError");
    Py_DECREF(pickle_module);
    if (unlikely(!pickle_error)) return;
    if (checksum2 == checksum1) {
        PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x) = (%s))",
            checksum, checksum1, members);
    } else if (checksum3 == checksum2) {
        PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x) = (%s))",
            checksum, checksum1, checksum2, members);
    } else {
        PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x, 0x%x) = (%s))",
            checksum, checksum1, checksum2, checksum3, members);
    }
    Py_DECREF(pickle_error);
}
static int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members) {
    int found = 0;
    found |= checksum1 == checksum;
    found |= checksum2 == checksum;
    found |= checksum3 == checksum;
    if (likely(found))
        return 0;
    __Pyx_RaiseUnpickleChecksumError(checksum, checksum1, checksum2, checksum3, members);
    return -1;
}

/* CIntFromPyVerify */
#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\
    __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0)
#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\
    __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1)
#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\
    {\
        func_type value = func_value;\
        if (sizeof(target_type) < sizeof(func_type)) {\
            if (unlikely(value != (func_type) (target_type) value)) {\
                func_type zero = 0;\
                if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\
                    return (target_type) -1;\
                if (is_unsigned && unlikely(value < zero))\
                    goto raise_neg_overflow;\
                else\
                    goto raise_overflow;\
            }\
        }\
        return (target_type) value;\
    }

/* CIntFromPy */
static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) {
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
    const long neg_one = (long) -1, const_zero = (long) 0;
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic pop
#endif
    const int is_unsigned = neg_one > const_zero;
    if (unlikely(!PyLong_Check(x))) {
        long val;
        PyObject *tmp = __Pyx_PyNumber_Long(x);
        if (!tmp) return (long) -1;
        val = __Pyx_PyLong_As_long(tmp);
        Py_DECREF(tmp);
        return val;
    }
    if (is_unsigned) {
#if CYTHON_USE_PYLONG_INTERNALS
        if (unlikely(__Pyx_PyLong_IsNeg(x))) {
            goto raise_neg_overflow;
        } else if (__Pyx_PyLong_IsCompact(x)) {
            __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x))
        } else {
            const digit* digits = __Pyx_PyLong_Digits(x);
            assert(__Pyx_PyLong_DigitCount(x) > 1);
            switch (__Pyx_PyLong_DigitCount(x)) {
                case 2:
                    if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) {
                            return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]));
                        }
                    }
                    break;
                case 3:
                    if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) {
                            return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]));
                        }
                    }
                    break;
                case 4:
                    if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) {
                            return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]));
                        }
                    }
                    break;
            }
        }
#endif
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7
        if (unlikely(Py_SIZE(x) < 0)) {
            goto raise_neg_overflow;
        }
#else
        {
            int result = PyObject_RichCompareBool(x, Py_False, Py_LT);
            if (unlikely(result < 0))
                return (long) -1;
            if (unlikely(result == 1))
                goto raise_neg_overflow;
        }
#endif
        if ((sizeof(long) <= sizeof(unsigned long))) {
            __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x))
        } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) {
            __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x))
        }
    } else {
#if CYTHON_USE_PYLONG_INTERNALS
        if (__Pyx_PyLong_IsCompact(x)) {
            __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x))
        } else {
            const digit* digits = __Pyx_PyLong_Digits(x);
            assert(__Pyx_PyLong_DigitCount(x) > 1);
            switch (__Pyx_PyLong_SignedDigitCount(x)) {
                case -2:
                    if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) {
                            return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
                        }
                    }
                    break;
                case 2:
                    if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) {
                            return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
                        }
                    }
                    break;
                case -3:
                    if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) {
                            return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
                        }
                    }
                    break;
                case 3:
                    if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) {
                            return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
                        }
                    }
                    break;
                case -4:
                    if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) {
                            return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
                        }
                    }
                    break;
                case 4:
                    if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) {
                            return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
                        }
                    }
                    break;
            }
        }
#endif
        if ((sizeof(long) <= sizeof(long))) {
            __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x))
        } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) {
            __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x))
        }
    }
    {
        long val;
        int ret = -1;
#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API
        Py_ssize_t bytes_copied = PyLong_AsNativeBytes(
            x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0));
        if (unlikely(bytes_copied == -1)) {
        } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) {
            goto raise_overflow;
        } else {
            ret = 0;
        }
#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray)
        int one = 1; int is_little = (int)*(unsigned char *)&one;
        unsigned char *bytes = (unsigned char *)&val;
        ret = _PyLong_AsByteArray((PyLongObject *)x,
                                    bytes, sizeof(val),
                                    is_little, !is_unsigned);
#else
        PyObject *v;
        PyObject *stepval = NULL, *mask = NULL, *shift = NULL;
        int bits, remaining_bits, is_negative = 0;
        int chunk_size = (sizeof(long) < 8) ? 30 : 62;
        if (likely(PyLong_CheckExact(x))) {
            v = __Pyx_NewRef(x);
        } else {
            v = PyNumber_Long(x);
            if (unlikely(!v)) return (long) -1;
            assert(PyLong_CheckExact(v));
        }
        {
            int result = PyObject_RichCompareBool(v, Py_False, Py_LT);
            if (unlikely(result < 0)) {
                Py_DECREF(v);
                return (long) -1;
            }
            is_negative = result == 1;
        }
        if (is_unsigned && unlikely(is_negative)) {
            Py_DECREF(v);
            goto raise_neg_overflow;
        } else if (is_negative) {
            stepval = PyNumber_Invert(v);
            Py_DECREF(v);
            if (unlikely(!stepval))
                return (long) -1;
        } else {
            stepval = v;
        }
        v = NULL;
        val = (long) 0;
        mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done;
        shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done;
        for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) {
            PyObject *tmp, *digit;
            long idigit;
            digit = PyNumber_And(stepval, mask);
            if (unlikely(!digit)) goto done;
            idigit = PyLong_AsLong(digit);
            Py_DECREF(digit);
            if (unlikely(idigit < 0)) goto done;
            val |= ((long) idigit) << bits;
            tmp = PyNumber_Rshift(stepval, shift);
            if (unlikely(!tmp)) goto done;
            Py_DECREF(stepval); stepval = tmp;
        }
        Py_DECREF(shift); shift = NULL;
        Py_DECREF(mask); mask = NULL;
        {
            long idigit = PyLong_AsLong(stepval);
            if (unlikely(idigit < 0)) goto done;
            remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1);
            if (unlikely(idigit >= (1L << remaining_bits)))
                goto raise_overflow;
            val |= ((long) idigit) << bits;
        }
        if (!is_unsigned) {
            if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1))))
                goto raise_overflow;
            if (is_negative)
                val = ~val;
        }
        ret = 0;
    done:
        Py_XDECREF(shift);
        Py_XDECREF(mask);
        Py_XDECREF(stepval);
#endif
        if (unlikely(ret))
            return (long) -1;
        return val;
    }
raise_overflow:
    PyErr_SetString(PyExc_OverflowError,
        "value too large to convert to long");
    return (long) -1;
raise_neg_overflow:
    PyErr_SetString(PyExc_OverflowError,
        "can't convert negative value to long");
    return (long) -1;
}

/* CIntToPy */
static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) {
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
    const int neg_one = (int) -1, const_zero = (int) 0;
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic pop
#endif
    const int is_unsigned = neg_one > const_zero;
    if (is_unsigned) {
        if (sizeof(int) < sizeof(long)) {
            return PyLong_FromLong((long) value);
        } else if (sizeof(int) <= sizeof(unsigned long)) {
            return PyLong_FromUnsignedLong((unsigned long) value);
#if !CYTHON_COMPILING_IN_PYPY
        } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) {
            return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value);
#endif
        }
    } else {
        if (sizeof(int) <= sizeof(long)) {
            return PyLong_FromLong((long) value);
        } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) {
            return PyLong_FromLongLong((PY_LONG_LONG) value);
        }
    }
    {
        unsigned char *bytes = (unsigned char *)&value;
#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4
        if (is_unsigned) {
            return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1);
        } else {
            return PyLong_FromNativeBytes(bytes, sizeof(value), -1);
        }
#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000
        int one = 1; int little = (int)*(unsigned char *)&one;
        return _PyLong_FromByteArray(bytes, sizeof(int),
                                     little, !is_unsigned);
#else
        int one = 1; int little = (int)*(unsigned char *)&one;
        PyObject *from_bytes, *result = NULL, *kwds = NULL;
        PyObject *py_bytes = NULL, *order_str = NULL;
        from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes");
        if (!from_bytes) return NULL;
        py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int));
        if (!py_bytes) goto limited_bad;
        order_str = PyUnicode_FromString(little ? "little" : "big");
        if (!order_str) goto limited_bad;
        {
            PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str };
            if (!is_unsigned) {
                kwds = __Pyx_MakeVectorcallBuilderKwds(1);
                if (!kwds) goto limited_bad;
                if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad;
            }
            result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds);
        }
        limited_bad:
        Py_XDECREF(kwds);
        Py_XDECREF(order_str);
        Py_XDECREF(py_bytes);
        Py_XDECREF(from_bytes);
        return result;
#endif
    }
}

/* CIntFromPy */
static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) {
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
    const int neg_one = (int) -1, const_zero = (int) 0;
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic pop
#endif
    const int is_unsigned = neg_one > const_zero;
    if (unlikely(!PyLong_Check(x))) {
        int val;
        PyObject *tmp = __Pyx_PyNumber_Long(x);
        if (!tmp) return (int) -1;
        val = __Pyx_PyLong_As_int(tmp);
        Py_DECREF(tmp);
        return val;
    }
    if (is_unsigned) {
#if CYTHON_USE_PYLONG_INTERNALS
        if (unlikely(__Pyx_PyLong_IsNeg(x))) {
            goto raise_neg_overflow;
        } else if (__Pyx_PyLong_IsCompact(x)) {
            __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x))
        } else {
            const digit* digits = __Pyx_PyLong_Digits(x);
            assert(__Pyx_PyLong_DigitCount(x) > 1);
            switch (__Pyx_PyLong_DigitCount(x)) {
                case 2:
                    if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) {
                            return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]));
                        }
                    }
                    break;
                case 3:
                    if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) {
                            return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]));
                        }
                    }
                    break;
                case 4:
                    if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) {
                            return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]));
                        }
                    }
                    break;
            }
        }
#endif
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7
        if (unlikely(Py_SIZE(x) < 0)) {
            goto raise_neg_overflow;
        }
#else
        {
            int result = PyObject_RichCompareBool(x, Py_False, Py_LT);
            if (unlikely(result < 0))
                return (int) -1;
            if (unlikely(result == 1))
                goto raise_neg_overflow;
        }
#endif
        if ((sizeof(int) <= sizeof(unsigned long))) {
            __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x))
        } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) {
            __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x))
        }
    } else {
#if CYTHON_USE_PYLONG_INTERNALS
        if (__Pyx_PyLong_IsCompact(x)) {
            __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x))
        } else {
            const digit* digits = __Pyx_PyLong_Digits(x);
            assert(__Pyx_PyLong_DigitCount(x) > 1);
            switch (__Pyx_PyLong_SignedDigitCount(x)) {
                case -2:
                    if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) {
                            return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
                        }
                    }
                    break;
                case 2:
                    if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) {
                            return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
                        }
                    }
                    break;
                case -3:
                    if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) {
                            return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
                        }
                    }
                    break;
                case 3:
                    if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) {
                            return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
                        }
                    }
                    break;
                case -4:
                    if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) {
                            return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
                        }
                    }
                    break;
                case 4:
                    if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) {
                        if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
                            __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
                        } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) {
                            return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
                        }
                    }
                    break;
            }
        }
#endif
        if ((sizeof(int) <= sizeof(long))) {
            __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x))
        } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) {
            __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x))
        }
    }
    {
        int val;
        int ret = -1;
#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API
        Py_ssize_t bytes_copied = PyLong_AsNativeBytes(
            x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0));
        if (unlikely(bytes_copied == -1)) {
        } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) {
            goto raise_overflow;
        } else {
            ret = 0;
        }
#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray)
        int one = 1; int is_little = (int)*(unsigned char *)&one;
        unsigned char *bytes = (unsigned char *)&val;
        ret = _PyLong_AsByteArray((PyLongObject *)x,
                                    bytes, sizeof(val),
                                    is_little, !is_unsigned);
#else
        PyObject *v;
        PyObject *stepval = NULL, *mask = NULL, *shift = NULL;
        int bits, remaining_bits, is_negative = 0;
        int chunk_size = (sizeof(long) < 8) ? 30 : 62;
        if (likely(PyLong_CheckExact(x))) {
            v = __Pyx_NewRef(x);
        } else {
            v = PyNumber_Long(x);
            if (unlikely(!v)) return (int) -1;
            assert(PyLong_CheckExact(v));
        }
        {
            int result = PyObject_RichCompareBool(v, Py_False, Py_LT);
            if (unlikely(result < 0)) {
                Py_DECREF(v);
                return (int) -1;
            }
            is_negative = result == 1;
        }
        if (is_unsigned && unlikely(is_negative)) {
            Py_DECREF(v);
            goto raise_neg_overflow;
        } else if (is_negative) {
            stepval = PyNumber_Invert(v);
            Py_DECREF(v);
            if (unlikely(!stepval))
                return (int) -1;
        } else {
            stepval = v;
        }
        v = NULL;
        val = (int) 0;
        mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done;
        shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done;
        for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) {
            PyObject *tmp, *digit;
            long idigit;
            digit = PyNumber_And(stepval, mask);
            if (unlikely(!digit)) goto done;
            idigit = PyLong_AsLong(digit);
            Py_DECREF(digit);
            if (unlikely(idigit < 0)) goto done;
            val |= ((int) idigit) << bits;
            tmp = PyNumber_Rshift(stepval, shift);
            if (unlikely(!tmp)) goto done;
            Py_DECREF(stepval); stepval = tmp;
        }
        Py_DECREF(shift); shift = NULL;
        Py_DECREF(mask); mask = NULL;
        {
            long idigit = PyLong_AsLong(stepval);
            if (unlikely(idigit < 0)) goto done;
            remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1);
            if (unlikely(idigit >= (1L << remaining_bits)))
                goto raise_overflow;
            val |= ((int) idigit) << bits;
        }
        if (!is_unsigned) {
            if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1))))
                goto raise_overflow;
            if (is_negative)
                val = ~val;
        }
        ret = 0;
    done:
        Py_XDECREF(shift);
        Py_XDECREF(mask);
        Py_XDECREF(stepval);
#endif
        if (unlikely(ret))
            return (int) -1;
        return val;
    }
raise_overflow:
    PyErr_SetString(PyExc_OverflowError,
        "value too large to convert to int");
    return (int) -1;
raise_neg_overflow:
    PyErr_SetString(PyExc_OverflowError,
        "can't convert negative value to int");
    return (int) -1;
}

/* CIntToPy */
static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) {
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
    const long neg_one = (long) -1, const_zero = (long) 0;
#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
#pragma GCC diagnostic pop
#endif
    const int is_unsigned = neg_one > const_zero;
    if (is_unsigned) {
        if (sizeof(long) < sizeof(long)) {
            return PyLong_FromLong((long) value);
        } else if (sizeof(long) <= sizeof(unsigned long)) {
            return PyLong_FromUnsignedLong((unsigned long) value);
#if !CYTHON_COMPILING_IN_PYPY
        } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) {
            return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value);
#endif
        }
    } else {
        if (sizeof(long) <= sizeof(long)) {
            return PyLong_FromLong((long) value);
        } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) {
            return PyLong_FromLongLong((PY_LONG_LONG) value);
        }
    }
    {
        unsigned char *bytes = (unsigned char *)&value;
#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4
        if (is_unsigned) {
            return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1);
        } else {
            return PyLong_FromNativeBytes(bytes, sizeof(value), -1);
        }
#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000
        int one = 1; int little = (int)*(unsigned char *)&one;
        return _PyLong_FromByteArray(bytes, sizeof(long),
                                     little, !is_unsigned);
#else
        int one = 1; int little = (int)*(unsigned char *)&one;
        PyObject *from_bytes, *result = NULL, *kwds = NULL;
        PyObject *py_bytes = NULL, *order_str = NULL;
        from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes");
        if (!from_bytes) return NULL;
        py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long));
        if (!py_bytes) goto limited_bad;
        order_str = PyUnicode_FromString(little ? "little" : "big");
        if (!order_str) goto limited_bad;
        {
            PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str };
            if (!is_unsigned) {
                kwds = __Pyx_MakeVectorcallBuilderKwds(1);
                if (!kwds) goto limited_bad;
                if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad;
            }
            result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds);
        }
        limited_bad:
        Py_XDECREF(kwds);
        Py_XDECREF(order_str);
        Py_XDECREF(py_bytes);
        Py_XDECREF(from_bytes);
        return result;
#endif
    }
}

/* PyObjectCallMethod1 */
#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)))
static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) {
    PyObject *result = __Pyx_PyObject_CallOneArg(method, arg);
    Py_DECREF(method);
    return result;
}
#endif
static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) {
#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))
    PyObject *args[2] = {obj, arg};
    (void) __Pyx_PyObject_CallOneArg;
    (void) __Pyx_PyObject_Call2Args;
    return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
#else
    PyObject *method = NULL, *result;
    int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method);
    if (likely(is_method)) {
        result = __Pyx_PyObject_Call2Args(method, obj, arg);
        Py_DECREF(method);
        return result;
    }
    if (unlikely(!method)) return NULL;
    return __Pyx__PyObject_CallMethod1(method, arg);
#endif
}

/* UpdateUnpickledDict */
static int __Pyx__UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) {
    PyObject *state_dict = __Pyx_PySequence_ITEM(state, index);
    if (unlikely(!state_dict)) {
        return -1;
    }
    int non_empty = PyObject_IsTrue(state_dict);
    if (non_empty == 0) {
        Py_DECREF(state_dict);
        return 0;
    } else if (unlikely(non_empty == -1)) {
        return -1;
    }
    PyObject *dict;
    #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
    dict = PyObject_GetAttrString(obj, "__dict__");
    #else
    dict = PyObject_GenericGetDict(obj, NULL);
    #endif
    if (unlikely(!dict)) {
        Py_DECREF(state_dict);
        return -1;
    }
    int result;
    if (likely(PyDict_CheckExact(dict))) {
        result = PyDict_Update(dict, state_dict);
    } else {
        PyObject *obj_result = __Pyx_PyObject_CallMethod1(dict, __pyx_mstate_global->__pyx_n_u_update, state_dict);
        if (likely(obj_result)) {
            Py_DECREF(obj_result);
            result = 0;
        } else {
            result = -1;
        }
    }
    Py_DECREF(state_dict);
    Py_DECREF(dict);
    return result;
}
static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) {
    Py_ssize_t state_size = __Pyx_PyTuple_GET_SIZE(state);
    #if !CYTHON_ASSUME_SAFE_SIZE
    if (unlikely(state_size == -1)) return -1;
    #endif
    if (state_size <= index) {
        return 0;
    }
    return __Pyx__UpdateUnpickledDict(obj, state, index);
}

/* FormatTypeName */
#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000
static __Pyx_TypeName
__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp)
{
    PyObject *module = NULL, *name = NULL, *result = NULL;
    #if __PYX_LIMITED_VERSION_HEX < 0x030b0000
    name = __Pyx_PyObject_GetAttrStr((PyObject *)tp,
                                               __pyx_mstate_global->__pyx_n_u_qualname);
    #else
    name = PyType_GetQualName(tp);
    #endif
    if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad;
    module = __Pyx_PyObject_GetAttrStr((PyObject *)tp,
                                               __pyx_mstate_global->__pyx_n_u_module);
    if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad;
    if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) {
        result = name;
        name = NULL;
        goto done;
    }
    result = PyUnicode_FromFormat("%U.%U", module, name);
    if (unlikely(result == NULL)) goto bad;
  done:
    Py_XDECREF(name);
    Py_XDECREF(module);
    return result;
  bad:
    PyErr_Clear();
    if (name) {
        result = name;
        name = NULL;
    } else {
        result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__7);
    }
    goto done;
}
#endif

/* GetRuntimeVersion */
#if __PYX_LIMITED_VERSION_HEX < 0x030b0000
void __Pyx_init_runtime_version(void) {
    if (__Pyx_cached_runtime_version == 0) {
        const char* rt_version = Py_GetVersion();
        unsigned long version = 0;
        unsigned long factor = 0x01000000UL;
        unsigned int digit = 0;
        int i = 0;
        while (factor) {
            while ('0' <= rt_version[i] && rt_version[i] <= '9') {
                digit = digit * 10 + (unsigned int) (rt_version[i] - '0');
                ++i;
            }
            version += factor * digit;
            if (rt_version[i] != '.')
                break;
            digit = 0;
            factor >>= 8;
            ++i;
        }
        __Pyx_cached_runtime_version = version;
    }
}
#endif
static unsigned long __Pyx_get_runtime_version(void) {
#if __PYX_LIMITED_VERSION_HEX >= 0x030b0000
    return Py_Version & ~0xFFUL;
#else
    return __Pyx_cached_runtime_version;
#endif
}

/* AddModuleRef */
#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
  static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) {
      PyObject *module_dict = PyImport_GetModuleDict();
      PyObject *m;
      if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) {
          return NULL;
      }
      if (m != NULL && PyModule_Check(m)) {
          return m;
      }
      Py_XDECREF(m);
      m = PyModule_NewObject(name);
      if (m == NULL)
          return NULL;
      if (PyDict_CheckExact(module_dict)) {
          PyObject *new_m;
          (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m);
          Py_DECREF(m);
          return new_m;
      } else {
           if (PyObject_SetItem(module_dict, name, m) != 0) {
                Py_DECREF(m);
                return NULL;
            }
            return m;
      }
  }
  static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) {
      PyObject *py_name = PyUnicode_FromString(name);
      if (!py_name) return NULL;
      PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name);
      Py_DECREF(py_name);
      return module;
  }
#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000
  #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name)
#else
  static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) {
      PyObject *module = PyImport_AddModule(name);
      Py_XINCREF(module);
      return module;
  }
#endif

/* CheckBinaryVersion */
static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) {
    const unsigned long MAJOR_MINOR = 0xFFFF0000UL;
    if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR))
        return 0;
    if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR)))
        return 1;
    {
        char message[200];
        PyOS_snprintf(message, sizeof(message),
                      "compile time Python version %d.%d "
                      "of module '%.100s' "
                      "%s "
                      "runtime version %d.%d",
                       (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF),
                       __Pyx_MODULE_NAME,
                       (allow_newer) ? "was newer than" : "does not match",
                       (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF)
       );
        return PyErr_WarnEx(NULL, message, 1);
    }
}

/* DecompressString */
static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) {
    PyObject *module = NULL, *decompress, *compressed_bytes, *decompressed;
    const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib";
    PyObject *methodname = PyUnicode_FromString("decompress");
    if (unlikely(!methodname)) return NULL;
    #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000
    if (algo == 3) {
        PyObject *fromlist = Py_BuildValue("[O]", methodname);
        if (unlikely(!fromlist)) goto bad;
        module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0);
        Py_DECREF(fromlist);
    } else
    #endif
        module = PyImport_ImportModule(module_name);
    if (unlikely(!module)) goto import_failed;
    decompress = PyObject_GetAttr(module, methodname);
    if (unlikely(!decompress)) goto import_failed;
    {
        #ifdef __cplusplus
            char *memview_bytes = const_cast<char*>(s);
        #else
            #if defined(__clang__)
              #pragma clang diagnostic push
              #pragma clang diagnostic ignored "-Wcast-qual"
            #elif !defined(__INTEL_COMPILER) && defined(__GNUC__)
              #pragma GCC diagnostic push
              #pragma GCC diagnostic ignored "-Wcast-qual"
            #endif
            char *memview_bytes = (char*) s;
            #if defined(__clang__)
              #pragma clang diagnostic pop
            #elif !defined(__INTEL_COMPILER) && defined(__GNUC__)
              #pragma GCC diagnostic pop
            #endif
        #endif
        #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ)
        int memview_flags = 0x100;
        #else
        int memview_flags = PyBUF_READ;
        #endif
        compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags);
    }
    if (unlikely(!compressed_bytes)) {
        Py_DECREF(decompress);
        goto bad;
    }
    decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL);
    Py_DECREF(compressed_bytes);
    Py_DECREF(decompress);
    Py_DECREF(module);
    Py_DECREF(methodname);
    return decompressed;
import_failed:
    PyErr_Format(PyExc_ImportError,
        "Failed to import '%.20s.decompress' - cannot initialise module strings. "
        "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.",
        module_name, algo);
bad:
    Py_XDECREF(module);
    Py_DECREF(methodname);
    return NULL;
}

#include <string.h>
static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) {
    size_t len = strlen(s);
    if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) {
        PyErr_SetString(PyExc_OverflowError, "byte string is too long");
        return -1;
    }
    return (Py_ssize_t) len;
}
static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) {
    Py_ssize_t len = __Pyx_ssize_strlen(c_str);
    if (unlikely(len < 0)) return NULL;
    return __Pyx_PyUnicode_FromStringAndSize(c_str, len);
}
static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) {
    Py_ssize_t len = __Pyx_ssize_strlen(c_str);
    if (unlikely(len < 0)) return NULL;
    return PyByteArray_FromStringAndSize(c_str, len);
}
static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) {
    Py_ssize_t ignore;
    return __Pyx_PyObject_AsStringAndSize(o, &ignore);
}
#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8
static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) {
    if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL;
#if CYTHON_COMPILING_IN_LIMITED_API
    {
        const char* result;
        Py_ssize_t unicode_length;
        CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
        #if __PYX_LIMITED_VERSION_HEX < 0x030A0000
        if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL;
        #else
        result = PyUnicode_AsUTF8AndSize(o, length);
        #endif
        #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
        unicode_length = PyUnicode_GetLength(o);
        if (unlikely(unicode_length < 0)) return NULL;
        if (unlikely(unicode_length != *length)) {
            PyUnicode_AsASCIIString(o);
            return NULL;
        }
        #endif
        return result;
    }
#else
#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
    if (likely(PyUnicode_IS_ASCII(o))) {
        *length = PyUnicode_GET_LENGTH(o);
        return PyUnicode_AsUTF8(o);
    } else {
        PyUnicode_AsASCIIString(o);
        return NULL;
    }
#else
    return PyUnicode_AsUTF8AndSize(o, length);
#endif
#endif
}
#endif
static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) {
#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8
    if (PyUnicode_Check(o)) {
        return __Pyx_PyUnicode_AsStringAndSize(o, length);
    } else
#endif
    if (PyByteArray_Check(o)) {
#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)))
        *length = PyByteArray_GET_SIZE(o);
        return PyByteArray_AS_STRING(o);
#else
        *length = PyByteArray_Size(o);
        if (*length == -1) return NULL;
        return PyByteArray_AsString(o);
#endif
    } else
    {
        char* result;
        int r = PyBytes_AsStringAndSize(o, &result, length);
        if (unlikely(r < 0)) {
            return NULL;
        } else {
            return result;
        }
    }
}
static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
   int is_true = x == Py_True;
   if (is_true | (x == Py_False) | (x == Py_None)) return is_true;
   else return PyObject_IsTrue(x);
}
static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) {
    int retval;
    if (unlikely(!x)) return -1;
    retval = __Pyx_PyObject_IsTrue(x);
    Py_DECREF(x);
    return retval;
}
static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) {
    __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result));
    if (PyLong_Check(result)) {
        if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
                "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ").  "
                "The ability to return an instance of a strict subclass of int is deprecated, "
                "and may be removed in a future version of Python.",
                result_type_name)) {
            __Pyx_DECREF_TypeName(result_type_name);
            Py_DECREF(result);
            return NULL;
        }
        __Pyx_DECREF_TypeName(result_type_name);
        return result;
    }
    PyErr_Format(PyExc_TypeError,
                 "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")",
                 result_type_name);
    __Pyx_DECREF_TypeName(result_type_name);
    Py_DECREF(result);
    return NULL;
}
static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) {
#if CYTHON_USE_TYPE_SLOTS
  PyNumberMethods *m;
#endif
  PyObject *res = NULL;
  if (likely(PyLong_Check(x)))
      return __Pyx_NewRef(x);
#if CYTHON_USE_TYPE_SLOTS
  m = Py_TYPE(x)->tp_as_number;
  if (likely(m && m->nb_int)) {
      res = m->nb_int(x);
  }
#else
  if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) {
      res = PyNumber_Long(x);
  }
#endif
  if (likely(res)) {
      if (unlikely(!PyLong_CheckExact(res))) {
          return __Pyx_PyNumber_LongWrongResultType(res);
      }
  }
  else if (!PyErr_Occurred()) {
      PyErr_SetString(PyExc_TypeError,
                      "an integer is required");
  }
  return res;
}
static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) {
  Py_ssize_t ival;
  PyObject *x;
  if (likely(PyLong_CheckExact(b))) {
    #if CYTHON_USE_PYLONG_INTERNALS
    if (likely(__Pyx_PyLong_IsCompact(b))) {
        return __Pyx_PyLong_CompactValue(b);
    } else {
      const digit* digits = __Pyx_PyLong_Digits(b);
      const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b);
      switch (size) {
         case 2:
           if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) {
             return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
           }
           break;
         case -2:
           if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) {
             return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
           }
           break;
         case 3:
           if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) {
             return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
           }
           break;
         case -3:
           if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) {
             return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
           }
           break;
         case 4:
           if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) {
             return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
           }
           break;
         case -4:
           if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) {
             return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
           }
           break;
      }
    }
    #endif
    return PyLong_AsSsize_t(b);
  }
  x = PyNumber_Index(b);
  if (!x) return -1;
  ival = PyLong_AsSsize_t(x);
  Py_DECREF(x);
  return ival;
}
static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) {
  if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) {
    return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o);
  } else {
    Py_ssize_t ival;
    PyObject *x;
    x = PyNumber_Index(o);
    if (!x) return -1;
    ival = PyLong_AsLong(x);
    Py_DECREF(x);
    return ival;
  }
}
static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) {
    CYTHON_UNUSED_VAR(b);
    return __Pyx_NewRef(Py_None);
}
static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) {
  return __Pyx_NewRef(b ? Py_True: Py_False);
}
static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) {
    return PyLong_FromSize_t(ival);
}


/* MultiPhaseInitModuleState */
#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE
#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000)
  #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1
#else
  #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0
#endif
#endif
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS
#error "Module state with PEP489 requires atomics. Currently that's one of\
 C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics"
#endif
#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
#define __Pyx_ModuleStateLookup_Lock()
#define __Pyx_ModuleStateLookup_Unlock()
#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000
static PyMutex __Pyx_ModuleStateLookup_mutex = {0};
#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex)
#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex)
#elif defined(__cplusplus) && __cplusplus >= 201103L
#include <mutex>
static std::mutex __Pyx_ModuleStateLookup_mutex;
#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock()
#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock()
#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__)
#include <threads.h>
static mtx_t __Pyx_ModuleStateLookup_mutex;
static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT;
static void __Pyx_ModuleStateLookup_initialize_mutex(void) {
    mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain);
}
#define __Pyx_ModuleStateLookup_Lock()\
  call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\
  mtx_lock(&__Pyx_ModuleStateLookup_mutex)
#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex)
#elif defined(HAVE_PTHREAD_H)
#include <pthread.h>
static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER;
#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex)
#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex)
#elif defined(_WIN32)
#include <Windows.h>  // synchapi.h on its own doesn't work
static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT;
#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex)
#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex)
#else
#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\
 Requires C standard >= C11, or C++ standard >= C++11,\
 or pthreads, or the Windows 32 API, or Python >= 3.13."
#endif
typedef struct {
    int64_t id;
    PyObject *module;
} __Pyx_InterpreterIdAndModule;
typedef struct {
    char interpreter_id_as_index;
    Py_ssize_t count;
    Py_ssize_t allocated;
    __Pyx_InterpreterIdAndModule table[1];
} __Pyx_ModuleStateLookupData;
#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0;
#endif
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0;
#else
static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL;
#endif
static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound(
        __Pyx_InterpreterIdAndModule* table,
        Py_ssize_t count,
        int64_t interpreterId) {
    __Pyx_InterpreterIdAndModule* begin = table;
    __Pyx_InterpreterIdAndModule* end = begin + count;
    if (begin->id == interpreterId) {
        return begin;
    }
    while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) {
        __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2;
        if (halfway->id == interpreterId) {
            return halfway;
        }
        if (halfway->id < interpreterId) {
            begin = halfway;
        } else {
            end = halfway;
        }
    }
    for (; begin < end; ++begin) {
        if (begin->id >= interpreterId) return begin;
    }
    return begin;
}
static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) {
    int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get());
    if (interpreter_id == -1) return NULL;
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
    __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data);
    {
        __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter);
        if (likely(data)) {
            __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data);
            if (likely(data == new_data)) {
                goto read_finished;
            }
        }
        __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter);
        __Pyx_ModuleStateLookup_Lock();
        __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter);
        data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data);
        __Pyx_ModuleStateLookup_Unlock();
    }
  read_finished:;
#else
    __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data;
#endif
    __Pyx_InterpreterIdAndModule* found = NULL;
    if (unlikely(!data)) goto end;
    if (data->interpreter_id_as_index) {
        if (interpreter_id < data->count) {
            found = data->table+interpreter_id;
        }
    } else {
        found = __Pyx_State_FindModuleStateLookupTableLowerBound(
            data->table, data->count, interpreter_id);
    }
  end:
    {
        PyObject *result=NULL;
        if (found && found->id == interpreter_id) {
            result = found->module;
        }
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
        __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter);
#endif
        return result;
    }
}
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) {
    while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0);
}
#else
#define __Pyx_ModuleStateLookup_wait_until_no_readers()
#endif
static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) {
    Py_ssize_t to_allocate = (*old_data)->allocated;
    while (to_allocate <= interpreter_id) {
        if (to_allocate == 0) to_allocate = 1;
        else to_allocate *= 2;
    }
    __Pyx_ModuleStateLookupData *new_data = *old_data;
    if (to_allocate != (*old_data)->allocated) {
         new_data = (__Pyx_ModuleStateLookupData *)realloc(
            *old_data,
            sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule));
        if (!new_data) {
            PyErr_NoMemory();
            return -1;
        }
        for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) {
            new_data->table[i].id = i;
            new_data->table[i].module = NULL;
        }
        new_data->allocated = to_allocate;
    }
    new_data->table[interpreter_id].module = module;
    if (new_data->count < interpreter_id+1) {
        new_data->count = interpreter_id+1;
    }
    *old_data = new_data;
    return 0;
}
static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) {
    __Pyx_InterpreterIdAndModule *read = data->table;
    __Pyx_InterpreterIdAndModule *write = data->table;
    __Pyx_InterpreterIdAndModule *end = read + data->count;
    for (; read<end; ++read) {
        if (read->module) {
            write->id = read->id;
            write->module = read->module;
            ++write;
        }
    }
    data->count = write - data->table;
    for (; write<end; ++write) {
        write->id = 0;
        write->module = NULL;
    }
    data->interpreter_id_as_index = 0;
}
static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) {
    int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get());
    if (interpreter_id == -1) return -1;
    int result = 0;
    __Pyx_ModuleStateLookup_Lock();
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
    __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *)
            __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0);
#else
    __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data;
#endif
    __Pyx_ModuleStateLookupData *new_data = old_data;
    if (!new_data) {
        new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData));
        if (!new_data) {
            result = -1;
            PyErr_NoMemory();
            goto end;
        }
        new_data->allocated = 1;
        new_data->interpreter_id_as_index = 1;
    }
    __Pyx_ModuleStateLookup_wait_until_no_readers();
    if (new_data->interpreter_id_as_index) {
        if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) {
            result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id);
            goto end;
        }
        __Pyx_State_ConvertFromInterpIdAsIndex(new_data);
    }
    {
        Py_ssize_t insert_at = 0;
        {
            __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound(
                new_data->table, new_data->count, interpreter_id);
            assert(lower_bound);
            insert_at = lower_bound - new_data->table;
            if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) {
                lower_bound->module = module;
                goto end;  // already in table, nothing more to do
            }
        }
        if (new_data->count+1 >= new_data->allocated) {
            Py_ssize_t to_allocate = (new_data->count+1)*2;
            new_data =
                (__Pyx_ModuleStateLookupData*)realloc(
                    new_data,
                    sizeof(__Pyx_ModuleStateLookupData) +
                    (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule));
            if (!new_data) {
                result = -1;
                new_data = old_data;
                PyErr_NoMemory();
                goto end;
            }
            new_data->allocated = to_allocate;
        }
        ++new_data->count;
        int64_t last_id = interpreter_id;
        PyObject *last_module = module;
        for (Py_ssize_t i=insert_at; i<new_data->count; ++i) {
            int64_t current_id = new_data->table[i].id;
            new_data->table[i].id = last_id;
            last_id = current_id;
            PyObject *current_module = new_data->table[i].module;
            new_data->table[i].module = last_module;
            last_module = current_module;
        }
    }
  end:
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
    __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data);
#else
    __Pyx_ModuleStateLookup_data = new_data;
#endif
    __Pyx_ModuleStateLookup_Unlock();
    return result;
}
static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) {
    int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get());
    if (interpreter_id == -1) return -1;
    __Pyx_ModuleStateLookup_Lock();
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
    __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *)
            __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0);
#else
    __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data;
#endif
    if (data->interpreter_id_as_index) {
        if (interpreter_id < data->count) {
            data->table[interpreter_id].module = NULL;
        }
        goto done;
    }
    {
        __Pyx_ModuleStateLookup_wait_until_no_readers();
        __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound(
            data->table, data->count, interpreter_id);
        if (!lower_bound) goto done;
        if (lower_bound->id != interpreter_id) goto done;
        __Pyx_InterpreterIdAndModule *end = data->table+data->count;
        for (;lower_bound<end-1; ++lower_bound) {
            lower_bound->id = (lower_bound+1)->id;
            lower_bound->module = (lower_bound+1)->module;
        }
    }
    --data->count;
    if (data->count == 0) {
        free(data);
        data = NULL;
    }
  done:
#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
    __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data);
#else
    __Pyx_ModuleStateLookup_data = data;
#endif
    __Pyx_ModuleStateLookup_Unlock();
    return 0;
}
#endif

/* #### Code section: utility_code_pragmas_end ### */
#ifdef _MSC_VER
#pragma warning( pop )
#endif



/* #### Code section: end ### */
#endif /* Py_PYTHON_H */
