From 789f968215923c1c06dd2e34de1b52e07bcdb2be Mon Sep 17 00:00:00 2001 From: Luc Berger-Vergiat Date: Wed, 6 Mar 2024 10:54:35 -0700 Subject: [PATCH 1/4] Sparse MKL: changing the location of the MKL_SAFE_CALL macro Moving the macro outside of namespaces to ensure that it will be interpreted correctly when called from any other location in the library. It does not make much sense to guard Impl code in the Experimental namespace and in this case it cleans up a problem with namespace disambiguation for the compiler... --- sparse/src/KokkosSparse_Utils_mkl.hpp | 6 ++++++ sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sparse/src/KokkosSparse_Utils_mkl.hpp b/sparse/src/KokkosSparse_Utils_mkl.hpp index 2f9437c650..a14e19f3cf 100644 --- a/sparse/src/KokkosSparse_Utils_mkl.hpp +++ b/sparse/src/KokkosSparse_Utils_mkl.hpp @@ -62,9 +62,15 @@ inline void mkl_internal_safe_call(sparse_status_t mkl_status, const char *name, } } +} // namespace Impl +} // namespace KokkosSparse + #define KOKKOSKERNELS_MKL_SAFE_CALL(call) \ KokkosSparse::Impl::mkl_internal_safe_call(call, #call, __FILE__, __LINE__) +namespace KokkosSparse { +namespace Impl { + inline sparse_operation_t mode_kk_to_mkl(char mode_kk) { switch (toupper(mode_kk)) { case 'N': return SPARSE_OPERATION_NON_TRANSPOSE; diff --git a/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp b/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp index 875913214c..37ccdaa324 100644 --- a/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp +++ b/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp @@ -26,13 +26,10 @@ #include namespace KokkosSparse { -namespace Experimental { namespace Impl { // MKL 2018 and above: use new interface: sparse_matrix_t and mkl_sparse_?_mv() -using KokkosSparse::Impl::mode_kk_to_mkl; - // Note: Scalar here is the Kokkos type, not the MKL type template inline void spmv_bsr_mkl(Handle* handle, sparse_operation_t op, Scalar alpha, @@ -338,7 +335,6 @@ KOKKOSSPARSE_SPMV_MV_MKL(Kokkos::complex, Kokkos::OpenMP, #undef KOKKOSSPARSE_SPMV_MV_MKL } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif // defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && (__INTEL_MKL__ > 2017) From dc8a1fdaf351d8db2344814ef9f1fb2ee8e5190a Mon Sep 17 00:00:00 2001 From: Luc Berger-Vergiat Date: Wed, 6 Mar 2024 16:44:49 -0700 Subject: [PATCH 2/4] Sparse BsrSpMV: removing Experimental namespace from Impl namespace --- sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp | 5 ----- sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp | 6 +----- sparse/src/KokkosSparse_spmv.hpp | 4 ++-- sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp | 4 ---- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp b/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp index 1c2e4f80e9..85e27f1b1b 100644 --- a/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp +++ b/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp @@ -27,7 +27,6 @@ #include namespace KokkosSparse { -namespace Experimental { namespace Impl { struct BsrMatrixSpMVTensorCoreFunctorParams { @@ -519,7 +518,6 @@ struct BsrMatrixSpMVTensorCoreDispatcher { }; } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif // #if CUDA && (VOLTA || AMPERE) @@ -537,7 +535,6 @@ struct BsrMatrixSpMVTensorCoreDispatcher { #include "KokkosKernels_ExecSpaceUtils.hpp" namespace KokkosSparse { -namespace Experimental { namespace Impl { namespace Bsr { @@ -1735,9 +1732,7 @@ void spMatMultiVec_transpose(const execution_space &exec, Handle *handle, /* ******************* */ } // namespace Bsr - } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif // KOKKOSSPARSE_IMPL_SPMV_BSRMATRIX_IMPL_HPP_ diff --git a/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp b/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp index cde7fc1461..1c8505d339 100644 --- a/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp +++ b/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp @@ -29,7 +29,6 @@ #endif namespace KokkosSparse { -namespace Experimental { namespace Impl { // default is no eti available @@ -48,7 +47,6 @@ struct spmv_mv_bsrmatrix_eti_spec_avail { }; } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #define KOKKOSSPARSE_SPMV_BSRMATRIX_ETI_SPEC_AVAIL( \ @@ -103,7 +101,6 @@ struct spmv_mv_bsrmatrix_eti_spec_avail { #include namespace KokkosSparse { -namespace Experimental { namespace Impl { // declaration @@ -218,7 +215,7 @@ struct SPMV_MV_BSRMATRIXalgo == SPMV_BSR_TC) method = Method::TensorCores; - if (!KokkosSparse::Experimental::Impl::TensorCoresAvailable< + if (!KokkosSparse::Impl::TensorCoresAvailable< ExecutionSpace, AMatrix, XVector, YVector>::value) { method = Method::Fallback; } @@ -365,7 +362,6 @@ struct SPMV_MV_BSRMATRIX::name() + "]"; Kokkos::Profiling::pushRegion(label); - Experimental::Impl::SPMV_BSRMATRIX< + Impl::SPMV_BSRMATRIX< ExecutionSpace, HandleImpl, AMatrix_Internal, XVector_Internal, YVector_Internal, false>::spmv_bsrmatrix(space, handle, mode, alpha, A_i, x_i, beta, y_i); Kokkos::Profiling::popRegion(); } else { - Experimental::Impl::SPMV_BSRMATRIX< + Impl::SPMV_BSRMATRIX< ExecutionSpace, HandleImpl, AMatrix_Internal, XVector_Internal, YVector_Internal>::spmv_bsrmatrix(space, handle, mode, alpha, A_i, x_i, beta, y_i); diff --git a/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp b/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp index 37ccdaa324..c1d4744920 100644 --- a/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp +++ b/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp @@ -355,7 +355,6 @@ KOKKOSSPARSE_SPMV_MV_MKL(Kokkos::complex, Kokkos::OpenMP, #include "KokkosSparse_Utils_cusparse.hpp" namespace KokkosSparse { -namespace Experimental { namespace Impl { template @@ -761,7 +760,6 @@ KOKKOSSPARSE_SPMV_MV_CUSPARSE(Kokkos::complex, int, int, #undef KOKKOSSPARSE_SPMV_MV_CUSPARSE } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif // (9000 <= CUDA_VERSION) @@ -777,7 +775,6 @@ KOKKOSSPARSE_SPMV_MV_CUSPARSE(Kokkos::complex, int, int, #include "KokkosSparse_Utils_rocsparse.hpp" namespace KokkosSparse { -namespace Experimental { namespace Impl { template @@ -1097,7 +1094,6 @@ KOKKOSSPARSE_SPMV_ROCSPARSE(Kokkos::complex, rocsparse_int, #undef KOKKOSSPARSE_SPMV_ROCSPARSE } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif // defined(KOKKOSKERNELS_ENABLE_TPL_ROCSPARSE) From 54afcb63336897524dbfdd5fb993f1428eb5febd Mon Sep 17 00:00:00 2001 From: Luc Berger-Vergiat Date: Wed, 6 Mar 2024 16:48:09 -0700 Subject: [PATCH 3/4] Applying clang-format --- .../impl/KokkosSparse_spmv_bsrmatrix_spec.hpp | 4 ++-- sparse/src/KokkosSparse_spmv.hpp | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp b/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp index 1c8505d339..7d4871446e 100644 --- a/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp +++ b/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp @@ -215,8 +215,8 @@ struct SPMV_MV_BSRMATRIXalgo == SPMV_BSR_TC) method = Method::TensorCores; - if (!KokkosSparse::Impl::TensorCoresAvailable< - ExecutionSpace, AMatrix, XVector, YVector>::value) { + if (!KokkosSparse::Impl::TensorCoresAvailable::value) { method = Method::Fallback; } // can't use tensor cores unless mode is no-transpose diff --git a/sparse/src/KokkosSparse_spmv.hpp b/sparse/src/KokkosSparse_spmv.hpp index eaeb07d766..160d9de0e6 100644 --- a/sparse/src/KokkosSparse_spmv.hpp +++ b/sparse/src/KokkosSparse_spmv.hpp @@ -337,16 +337,17 @@ void spmv(const ExecutionSpace& space, Handle* handle, const char mode[], typename AMatrix_Internal::non_const_value_type>::name() + "]"; Kokkos::Profiling::pushRegion(label); - Impl::SPMV_BSRMATRIX< - ExecutionSpace, HandleImpl, AMatrix_Internal, XVector_Internal, - YVector_Internal, false>::spmv_bsrmatrix(space, handle, mode, alpha, - A_i, x_i, beta, y_i); + Impl::SPMV_BSRMATRIX::spmv_bsrmatrix(space, handle, mode, alpha, + A_i, x_i, beta, y_i); Kokkos::Profiling::popRegion(); } else { - Impl::SPMV_BSRMATRIX< - ExecutionSpace, HandleImpl, AMatrix_Internal, XVector_Internal, - YVector_Internal>::spmv_bsrmatrix(space, handle, mode, alpha, A_i, - x_i, beta, y_i); + Impl::SPMV_BSRMATRIX::spmv_bsrmatrix(space, handle, + mode, alpha, A_i, + x_i, beta, y_i); } } else { ///////////////// From 0cf6093b727abd6d2be337839db54c279e02a194 Mon Sep 17 00:00:00 2001 From: Luc Berger-Vergiat Date: Wed, 6 Mar 2024 17:32:39 -0700 Subject: [PATCH 4/4] Sparse SpMV: fixing more namespace issues! --- .../spmv/KokkosSparse_spmv_bsrmatrix_eti_spec_inst.cpp.in | 4 +--- .../KokkosSparse_spmv_mv_bsrmatrix_eti_spec_inst.cpp.in | 2 -- .../KokkosSparse_spmv_bsrmatrix_eti_spec_avail.hpp.in | 2 -- .../KokkosSparse_spmv_mv_bsrmatrix_eti_spec_avail.hpp.in | 2 -- sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp | 6 +++--- sparse/src/KokkosSparse_spmv.hpp | 4 ++-- sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_avail.hpp | 2 -- 7 files changed, 6 insertions(+), 16 deletions(-) diff --git a/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_bsrmatrix_eti_spec_inst.cpp.in b/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_bsrmatrix_eti_spec_inst.cpp.in index 9895083764..077150f36c 100644 --- a/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_bsrmatrix_eti_spec_inst.cpp.in +++ b/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_bsrmatrix_eti_spec_inst.cpp.in @@ -19,11 +19,9 @@ #include "KokkosSparse_spmv_bsrmatrix_spec.hpp" namespace KokkosSparse { -namespace Experimental { namespace Impl { // clang-format off @SPARSE_SPMV_BSRMATRIX_ETI_INST_BLOCK@ // clang-format on } // namespace Impl -} // namespace Experimental -} // namespace KokkosSparse \ No newline at end of file +} // namespace KokkosSparse diff --git a/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_inst.cpp.in b/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_inst.cpp.in index d089eca0e3..2c9a6083bf 100644 --- a/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_inst.cpp.in +++ b/sparse/eti/generated_specializations_cpp/spmv/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_inst.cpp.in @@ -19,11 +19,9 @@ #include "KokkosSparse_spmv_bsrmatrix_spec.hpp" namespace KokkosSparse { -namespace Experimental { namespace Impl { // clang-format off @SPARSE_SPMV_MV_BSRMATRIX_ETI_INST_BLOCK@ /// // clang-format on } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse diff --git a/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_bsrmatrix_eti_spec_avail.hpp.in b/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_bsrmatrix_eti_spec_avail.hpp.in index f98e60ae0d..278b60a813 100644 --- a/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_bsrmatrix_eti_spec_avail.hpp.in +++ b/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_bsrmatrix_eti_spec_avail.hpp.in @@ -17,12 +17,10 @@ #ifndef KOKKOSSPARSE_SPMV_BSRMATRIX_ETI_SPEC_AVAIL_HPP_ #define KOKKOSSPARSE_SPMV_BSRMATRIX_ETI_SPEC_AVAIL_HPP_ namespace KokkosSparse { -namespace Experimental { namespace Impl { // clang-format off @SPARSE_SPMV_BSRMATRIX_ETI_AVAIL_BLOCK@ // clang-format on } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif diff --git a/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_avail.hpp.in b/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_avail.hpp.in index df53928266..3247985f4c 100644 --- a/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_avail.hpp.in +++ b/sparse/eti/generated_specializations_hpp/KokkosSparse_spmv_mv_bsrmatrix_eti_spec_avail.hpp.in @@ -18,12 +18,10 @@ #define KOKKOSSPARSE_SPMV_MV_BSRMATRIX_ETI_SPEC_AVAIL_HPP_ namespace KokkosSparse { -namespace Experimental { namespace Impl { // clang-format off @SPARSE_SPMV_MV_BSRMATRIX_ETI_AVAIL_BLOCK@ // clang-format on } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif diff --git a/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp b/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp index 7d4871446e..5ec3cdb50b 100644 --- a/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp +++ b/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp @@ -46,9 +46,6 @@ struct spmv_mv_bsrmatrix_eti_spec_avail { enum : bool { value = false }; }; -} // namespace Impl -} // namespace KokkosSparse - #define KOKKOSSPARSE_SPMV_BSRMATRIX_ETI_SPEC_AVAIL( \ SCALAR_TYPE, ORDINAL_TYPE, OFFSET_TYPE, LAYOUT_TYPE, EXEC_SPACE_TYPE, \ MEM_SPACE_TYPE) \ @@ -95,6 +92,9 @@ struct spmv_mv_bsrmatrix_eti_spec_avail { enum : bool { value = true }; \ }; +} // namespace Impl +} // namespace KokkosSparse + // Include which ETIs are available #include #include diff --git a/sparse/src/KokkosSparse_spmv.hpp b/sparse/src/KokkosSparse_spmv.hpp index 160d9de0e6..e9a7a5b5e9 100644 --- a/sparse/src/KokkosSparse_spmv.hpp +++ b/sparse/src/KokkosSparse_spmv.hpp @@ -377,7 +377,7 @@ void spmv(const ExecutionSpace& space, Handle* handle, const char mode[], typename AMatrix_Internal::non_const_value_type>::name() + "]"; Kokkos::Profiling::pushRegion(label); - Experimental::Impl::SPMV_MV_BSRMATRIX< + Impl::SPMV_MV_BSRMATRIX< ExecutionSpace, HandleImpl, AMatrix_Internal, XVector_Internal, YVector_Internal, std::is_integral< @@ -386,7 +386,7 @@ void spmv(const ExecutionSpace& space, Handle* handle, const char mode[], beta, y_i); Kokkos::Profiling::popRegion(); } else { - Experimental::Impl::SPMV_MV_BSRMATRIX< + Impl::SPMV_MV_BSRMATRIX< ExecutionSpace, HandleImpl, AMatrix_Internal, XVector_Internal, YVector_Internal, std::is_integral:: diff --git a/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_avail.hpp b/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_avail.hpp index 3a68ba348e..16bf1abecf 100644 --- a/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_avail.hpp +++ b/sparse/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_avail.hpp @@ -22,7 +22,6 @@ #endif namespace KokkosSparse { -namespace Experimental { namespace Impl { // Specialization struct which defines whether a specialization exists template , #endif // defined(KOKKOSKERNELS_ENABLE_TPL_ROCSPARSE) } // namespace Impl -} // namespace Experimental } // namespace KokkosSparse #endif // KOKKOSPARSE_SPMV_BSRMATRIX_TPL_SPEC_AVAIL_HPP_