30 #define SACADO_VIEW_CUDA_HIERARCHICAL_DFAD 1 31 #define SACADO_KOKKOS_USE_MEMORY_POOL 1 37 #include "impl/Kokkos_Timer.hpp" 39 template <
typename ViewTypeA,
typename ViewTypeB,
typename ViewTypeC>
42 typedef typename ViewTypeC::value_type scalar_type;
43 typedef typename ViewTypeC::execution_space execution_space;
45 #if defined (KOKKOS_ENABLE_CUDA) 48 const bool is_cuda =
false;
50 const unsigned vector_size = is_cuda ? 32 : 1;
51 const unsigned team_size = is_cuda ? 128 / vector_size : 1;
53 const int m =
A.extent(0);
54 const int n =
A.extent(1);
55 const int range = (m+team_size-1)/team_size;
57 typedef Kokkos::TeamPolicy<execution_space> Policy;
59 Policy( range,team_size,vector_size ),
60 KOKKOS_LAMBDA (
const typename Policy::member_type& team) {
61 const int i = team.league_rank()*team.team_size() + team.team_rank();
66 for (
int j=0; j<n; ++j)
73 template <
typename ViewTypeA,
typename ViewTypeB,
typename ViewTypeC>
75 const ViewTypeA&
A,
const ViewTypeB& b,
const ViewTypeC&
c) {
76 typedef typename ViewTypeC::value_type scalar_type;
77 typedef typename ViewTypeC::execution_space execution_space;
78 typedef Kokkos::TeamPolicy<execution_space> Policy;
79 typedef typename Policy::member_type team_member;
80 typedef Kokkos::View<scalar_type*,Kokkos::LayoutLeft, typename execution_space::scratch_memory_space, Kokkos::MemoryUnmanaged> TmpScratchSpace;
82 #if defined (KOKKOS_ENABLE_CUDA) 85 const bool is_cuda =
false;
87 const unsigned VectorSize = is_cuda ? 32 : 1;
88 const unsigned TeamSize = is_cuda ? 128 / VectorSize : 1;
90 const int m =
A.extent(0);
91 const int n =
A.extent(1);
92 const int p = dimension_scalar(
A);
93 const int N = (m+TeamSize-1)/TeamSize;
95 Policy policy(
N, TeamSize, VectorSize);
96 const size_t bytes = TmpScratchSpace::shmem_size(TeamSize,
p);
98 policy.set_scratch_size(0, Kokkos::PerTeam(bytes)),
99 KOKKOS_LAMBDA (
const team_member& team) {
100 const int team_rank = team.team_rank();
101 const int team_size = team.team_size();
102 TmpScratchSpace t(team.team_scratch(0), team_size,
p);
103 const int i = team.league_rank()*team_size + team_rank;
106 for (
int j=0; j<n; ++j)
107 t(team_rank) +=
A(
i,j)*b(j);
114 template <
typename ViewTypeA,
typename ViewTypeB,
typename ViewTypeC>
118 const double tol = 1.0e-14;
119 typedef typename ViewTypeC::value_type value_type;
120 typename ViewTypeC::HostMirror h_c = Kokkos::create_mirror_view(
c);
121 Kokkos::deep_copy(h_c,
c);
122 const size_t m =
A.extent(0);
123 const size_t n =
A.extent(1);
124 const size_t p = Kokkos::dimension_scalar(
A);
125 for (
size_t i=0;
i<m; ++
i) {
126 for (
size_t j=0; j<
p; ++j) {
127 value_type t = (j ==
p-1 ? n : 2*n);
129 std::cout <<
"Comparison failed! " <<
i <<
"," << j <<
" : " 130 << h_c(
i).fastAccessDx(j) <<
" , " << t << std::endl;
136 template <
typename FadType,
typename ... ViewArgs>
139 const size_t nloop,
const bool check)
141 typedef Kokkos::View<
FadType**, ViewArgs...> ViewTypeA;
142 typedef Kokkos::View<
FadType*, ViewArgs...> ViewTypeB;
143 typedef Kokkos::View<
FadType*, ViewArgs...> ViewTypeC;
144 typedef typename ViewTypeA::execution_space execution_space;
148 typedef Kokkos::View<FadType**, ConLayoutA, execution_space> ConViewTypeA;
149 typedef Kokkos::View<FadType*, ConLayoutB, execution_space> ConViewTypeB;
150 typedef Kokkos::View<FadType*, ConLayoutC, execution_space> ConViewTypeC;
152 ConViewTypeA
A(
"A",m,n,
p+1);
153 ConViewTypeB b(
"B",n,
p+1);
154 ConViewTypeC
c(
"c",m,
p+1);
159 Kokkos::deep_copy(
typename ConViewTypeA::array_type(
A), 1.0);
160 Kokkos::deep_copy(
typename ConViewTypeB::array_type(b), 1.0);
162 Kokkos::Impl::Timer wall_clock;
165 #if defined (KOKKOS_ENABLE_CUDA) 168 const bool is_cuda =
false;
170 const size_t concurrency = execution_space::concurrency();
171 const size_t warp_dim = is_cuda ? 32 : 1;
172 const size_t block_size =
p*
sizeof(double);
173 const size_t nkernels = concurrency / warp_dim;
174 const size_t mem_pool_size =
175 static_cast<size_t>(1.2*nkernels*block_size);
176 const size_t superblock_size = std::max<size_t>(nkernels / 100, 1) * block_size;
177 execution_space space;
186 execution_space().fence();
189 for (
size_t l=0; l<nloop; l++) {
192 execution_space().fence();
194 perf.
time = wall_clock.seconds() / nloop;
207 template <
typename FadType,
typename ... ViewArgs>
210 const size_t m,
const size_t n,
const size_t p,
const size_t nloop,
213 typedef Kokkos::View<
FadType**, ViewArgs...> ViewTypeA;
214 typedef Kokkos::View<
FadType*, ViewArgs...> ViewTypeB;
215 typedef Kokkos::View<
FadType*, ViewArgs...> ViewTypeC;
216 typedef typename ViewTypeA::execution_space execution_space;
220 typedef Kokkos::View<FadType**, ConLayoutA, execution_space> ConViewTypeA;
221 typedef Kokkos::View<FadType*, ConLayoutB, execution_space> ConViewTypeB;
222 typedef Kokkos::View<FadType*, ConLayoutC, execution_space> ConViewTypeC;
224 ConViewTypeA
A(
"A",m,n,
p+1);
225 ConViewTypeB b(
"B",n,
p+1);
226 ConViewTypeC
c(
"c",m,
p+1);
231 Kokkos::deep_copy(
typename ConViewTypeA::array_type(
A), 1.0);
232 Kokkos::deep_copy(
typename ConViewTypeB::array_type(b), 1.0);
234 Kokkos::Impl::Timer wall_clock;
239 execution_space().fence();
242 for (
size_t l=0; l<nloop; l++) {
245 execution_space().fence();
247 perf.
time = wall_clock.seconds() / nloop;
260 #define INST_FUNC_FAD_DEV(FAD,DEV) \ 261 template Perf do_time_fad_hierarchical_dfad< FAD, Kokkos::LayoutLeft, DEV > ( const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check ); \ 262 template Perf do_time_fad_hierarchical_dfad< FAD, Kokkos::LayoutRight, DEV > ( const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check ); \ 263 template Perf do_time_fad_hierarchical_dfad< FAD, DEV > ( const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check ); \ 264 template Perf do_time_fad_hierarchical_dfad_scratch< FAD, Kokkos::LayoutLeft, DEV > ( const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check ); \ 265 template Perf do_time_fad_hierarchical_dfad_scratch< FAD, Kokkos::LayoutRight, DEV > ( const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check ); \ 266 template Perf do_time_fad_hierarchical_dfad_scratch< FAD, DEV > ( const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check ); 268 #define INST_FUNC_DEV(DEV) \ 269 INST_FUNC_FAD_DEV( DFad_type, DEV ) 271 #ifdef KOKKOS_ENABLE_SERIAL 275 #ifdef KOKKOS_ENABLE_OPENMP 279 #ifdef KOKKOS_ENABLE_THREADS 283 #ifdef KOKKOS_ENABLE_CUDA
std::enable_if< !Kokkos::is_view_fad< View2 >::value, bool >::type check(const View1 &v_gold, const View2 &v, const double tol)
void createGlobalMemoryPool(const ExecSpace &space, const size_t min_total_alloc_size, const uint32_t min_block_alloc_size, const uint32_t max_block_alloc_size, const uint32_t min_superblock_size)
void check_deriv_hierarchical_dfad(const ViewTypeA &A, const ViewTypeB &b, const ViewTypeC &c)
Sacado::Fad::DFad< double > FadType
Perf do_time_fad_hierarchical_dfad_scratch(const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check)
expr expr1 expr1 expr1 c expr2 expr1 expr2 expr1 expr2 expr1 expr1 expr1 expr1 c expr2 expr1 expr2 expr1 expr2 expr1 expr1 expr1 expr1 c *expr2 expr1 expr2 expr1 expr2 expr1 expr1 expr1 expr1 c expr2 expr1 expr2 expr1 expr2 expr1 expr1 expr1 expr2 expr1 expr2 expr1 expr1 expr1 expr2 expr1 expr2 expr1 expr1 expr1 c
Perf do_time_fad_hierarchical_dfad(const size_t m, const size_t n, const size_t p, const size_t nloop, const bool check)
#define INST_FUNC_DEV(DEV)
expr expr expr fastAccessDx(i)) FAD_UNARYOP_MACRO(exp
void run_mat_vec_hierarchical_dfad_scratch(const ViewTypeA &A, const ViewTypeB &b, const ViewTypeC &c)
Sacado::Fad::DFad< double > DFad_type
void destroyGlobalMemoryPool(const ExecSpace &space)
void run_mat_vec_hierarchical_dfad(const ViewTypeA &A, const ViewTypeB &b, const ViewTypeC &c)