53 #ifndef AMESOS2_TPETRA_MULTIVEC_ADAPTER_DEF_HPP 54 #define AMESOS2_TPETRA_MULTIVEC_ADAPTER_DEF_HPP 56 #include <type_traits> 63 using Tpetra::MultiVector;
65 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
70 Node> >::MultiVecAdapter(
const Teuchos::RCP<multivec_t>& m )
75 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
76 typename MultiVecAdapter<
80 Node> >::multivec_t::impl_scalar_type *
85 Node> >::getMVPointer_impl()
const 87 TEUCHOS_TEST_FOR_EXCEPTION( this->getGlobalNumVectors() != 1,
88 std::invalid_argument,
89 "Amesos2_TpetraMultiVectorAdapter: getMVPointer_impl should only be called for case with a single vector and single MPI process" );
91 auto contig_local_view_2d = mv_->getLocalViewHost(Tpetra::Access::ReadWrite);
92 auto contig_local_view_1d = Kokkos::subview (contig_local_view_2d, Kokkos::ALL (), 0);
93 return contig_local_view_1d.data();
104 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
110 Node> >::get1dCopy(
const Teuchos::ArrayView<scalar_t>& av,
113 const Tpetra::Map<LocalOrdinal,
115 Node> > distribution_map,
120 typedef Tpetra::Map<LocalOrdinal, GlobalOrdinal, Node> map_type;
121 const size_t num_vecs = getGlobalNumVectors ();
123 TEUCHOS_TEST_FOR_EXCEPTION(
124 distribution_map.getRawPtr () == NULL, std::invalid_argument,
125 "Amesos2::MultiVecAdapter::get1dCopy: distribution_map argument is null.");
126 TEUCHOS_TEST_FOR_EXCEPTION(
127 mv_.is_null (), std::logic_error,
128 "Amesos2::MultiVecAdapter::get1dCopy: mv_ is null.");
130 TEUCHOS_TEST_FOR_EXCEPTION(
131 this->getMap ().is_null (), std::logic_error,
132 "Amesos2::MultiVecAdapter::get1dCopy: this->getMap() returns null.");
134 #ifdef HAVE_AMESOS2_DEBUG 135 const size_t requested_vector_length = distribution_map->getNodeNumElements ();
136 TEUCHOS_TEST_FOR_EXCEPTION(
137 lda < requested_vector_length, std::invalid_argument,
138 "Amesos2::MultiVecAdapter::get1dCopy: On process " <<
139 distribution_map->getComm ()->getRank () <<
" of the distribution Map's " 140 "communicator, the given stride lda = " << lda <<
" is not large enough " 141 "for the local vector length " << requested_vector_length <<
".");
142 TEUCHOS_TEST_FOR_EXCEPTION(
143 as<size_t> (av.size ()) < as<size_t> ((num_vecs - 1) * lda + requested_vector_length),
144 std::invalid_argument,
"Amesos2::MultiVector::get1dCopy: MultiVector " 145 "storage not large enough given leading dimension and number of vectors." );
146 #endif // HAVE_AMESOS2_DEBUG 149 if ( num_vecs == 1 && this->getComm()->getRank() == 0 && this->getComm()->getSize() == 1 ) {
150 mv_->get1dCopy (av, lda);
157 RCP<const map_type> distMap;
158 if (exporter_.is_null () ||
159 ! exporter_->getSourceMap ()->isSameAs (* (this->getMap ())) ||
160 ! exporter_->getTargetMap ()->isSameAs (* distribution_map)) {
166 distMap = rcp(
new map_type(*distribution_map));
168 exporter_ = rcp (
new export_type (this->getMap (), distMap));
171 distMap = exporter_->getTargetMap ();
174 multivec_t redist_mv (distMap, num_vecs);
177 redist_mv.doExport (*mv_, *exporter_, Tpetra::REPLACE);
179 if ( distribution != CONTIGUOUS_AND_ROOTED ) {
182 redist_mv.get1dCopy (av, lda);
188 auto contig_local_view_2d = redist_mv.getLocalViewHost(Tpetra::Access::ReadOnly);
189 if ( redist_mv.isConstantStride() ) {
190 for (
size_t j = 0; j < num_vecs; ++j) {
191 auto av_j = av(lda*j, lda);
192 for (
size_t i = 0; i < lda; ++i ) {
193 av_j[i] = contig_local_view_2d(i,j);
202 const size_t lclNumRows = redist_mv.getLocalLength();
203 for (
size_t j = 0; j < redist_mv.getNumVectors(); ++j) {
204 auto av_j = av(lda*j, lclNumRows);
205 auto X_lcl_j_2d = redist_mv.getLocalViewHost(Tpetra::Access::ReadOnly);
206 auto X_lcl_j_1d = Kokkos::subview (X_lcl_j_2d, Kokkos::ALL (), j);
208 using val_type =
typename std::remove_const<typename decltype( X_lcl_j_1d )::value_type>::type;
209 Kokkos::View<val_type*, Kokkos::HostSpace> umavj ( const_cast< val_type* > ( reinterpret_cast<const val_type*> ( av_j.getRawPtr () ) ), av_j.size () );
210 Kokkos::deep_copy (umavj, X_lcl_j_1d);
217 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
218 template <
typename KV>
224 Node> >::get1dCopy_kokkos_view(
229 const Tpetra::Map<LocalOrdinal,
231 Node> > distribution_map,
236 typedef Tpetra::Map<LocalOrdinal, GlobalOrdinal, Node> map_type;
237 const size_t num_vecs = getGlobalNumVectors ();
239 TEUCHOS_TEST_FOR_EXCEPTION(
240 distribution_map.getRawPtr () == NULL, std::invalid_argument,
241 "Amesos2::MultiVecAdapter::get1dCopy_kokkos_view: distribution_map argument is null.");
242 TEUCHOS_TEST_FOR_EXCEPTION(
243 mv_.is_null (), std::logic_error,
244 "Amesos2::MultiVecAdapter::get1dCopy_kokkos_view: mv_ is null.");
246 TEUCHOS_TEST_FOR_EXCEPTION(
247 this->getMap ().is_null (), std::logic_error,
248 "Amesos2::MultiVecAdapter::get1dCopy_kokkos_view: this->getMap() returns null.");
250 #ifdef HAVE_AMESOS2_DEBUG 251 const size_t requested_vector_length = distribution_map->getNodeNumElements ();
252 TEUCHOS_TEST_FOR_EXCEPTION(
253 lda < requested_vector_length, std::invalid_argument,
254 "Amesos2::MultiVecAdapter::get1dCopy_kokkos_view: On process " <<
255 distribution_map->getComm ()->getRank () <<
" of the distribution Map's " 256 "communicator, the given stride lda = " << lda <<
" is not large enough " 257 "for the local vector length " << requested_vector_length <<
".");
261 #endif // HAVE_AMESOS2_DEBUG 264 if ( num_vecs == 1 && this->getComm()->getRank() == 0 && this->getComm()->getSize() == 1 ) {
265 if(mv_->isConstantStride()) {
268 deep_copy_only(bInitialize, kokkos_view, mv_->getLocalViewDevice(Tpetra::Access::ReadOnly), bAssigned);
272 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::runtime_error,
"Resolve handling for non-constant stride.");
280 RCP<const map_type> distMap;
281 if (exporter_.is_null () ||
282 ! exporter_->getSourceMap ()->isSameAs (* (this->getMap ())) ||
283 ! exporter_->getTargetMap ()->isSameAs (* distribution_map)) {
289 distMap = rcp(
new map_type(*distribution_map));
291 exporter_ = rcp (
new export_type (this->getMap (), distMap));
294 distMap = exporter_->getTargetMap ();
297 multivec_t redist_mv (distMap, num_vecs);
300 redist_mv.doExport (*mv_, *exporter_, Tpetra::REPLACE);
302 if ( distribution != CONTIGUOUS_AND_ROOTED ) {
306 deep_copy_or_assign_view(bInitialize, kokkos_view, redist_mv.getLocalViewDevice(Tpetra::Access::ReadOnly), bAssigned);
310 if(redist_mv.isConstantStride()) {
312 deep_copy_or_assign_view(bInitialize, kokkos_view, redist_mv.getLocalViewDevice(Tpetra::Access::ReadOnly), bAssigned);
316 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::runtime_error,
"Kokkos adapter non-constant stride not imlemented.");
322 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
323 Teuchos::ArrayRCP<Scalar>
328 Node> >::get1dViewNonConst (
bool local)
334 TEUCHOS_TEST_FOR_EXCEPTION(
335 true, std::logic_error,
"Amesos2::MultiVecAdapter::get1dViewNonConst: " 389 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node>
395 Node> >::put1dData(
const Teuchos::ArrayView<const scalar_t>& new_data,
398 const Tpetra::Map<LocalOrdinal,
404 typedef Tpetra::Map<LocalOrdinal, GlobalOrdinal, Node> map_type;
406 TEUCHOS_TEST_FOR_EXCEPTION(
407 source_map.getRawPtr () == NULL, std::invalid_argument,
408 "Amesos2::MultiVecAdapter::put1dData: source_map argument is null.");
409 TEUCHOS_TEST_FOR_EXCEPTION(
410 mv_.is_null (), std::logic_error,
411 "Amesos2::MultiVecAdapter::put1dData: the internal MultiVector mv_ is null.");
413 TEUCHOS_TEST_FOR_EXCEPTION(
414 this->getMap ().is_null (), std::logic_error,
415 "Amesos2::MultiVecAdapter::put1dData: this->getMap() returns null.");
417 const size_t num_vecs = getGlobalNumVectors ();
420 if ( num_vecs == 1 && this->getComm()->getRank() == 0 && this->getComm()->getSize() == 1 ) {
422 auto mv_view_to_modify_2d = mv_->getLocalViewHost(Tpetra::Access::OverwriteAll);
423 for (
size_t i = 0; i < lda; ++i ) {
424 mv_view_to_modify_2d(i,0) = new_data[i];
432 RCP<const map_type> srcMap;
433 if (importer_.is_null () ||
434 ! importer_->getSourceMap ()->isSameAs (* source_map) ||
435 ! importer_->getTargetMap ()->isSameAs (* (this->getMap ()))) {
441 srcMap = rcp(
new map_type(*source_map));
442 importer_ = rcp (
new import_type (srcMap, this->getMap ()));
445 srcMap = importer_->getSourceMap ();
448 if ( distribution != CONTIGUOUS_AND_ROOTED ) {
451 const multivec_t source_mv (srcMap, new_data, lda, num_vecs);
452 mv_->doImport (source_mv, *importer_, Tpetra::REPLACE);
455 multivec_t redist_mv (srcMap, num_vecs);
456 if ( redist_mv.isConstantStride() ) {
457 auto contig_local_view_2d = redist_mv.getLocalViewHost(Tpetra::Access::OverwriteAll);
458 for (
size_t j = 0; j < num_vecs; ++j) {
459 auto av_j = new_data(lda*j, lda);
460 for (
size_t i = 0; i < lda; ++i ) {
461 contig_local_view_2d(i,j) = av_j[i];
470 const size_t lclNumRows = redist_mv.getLocalLength();
471 for (
size_t j = 0; j < redist_mv.getNumVectors(); ++j) {
472 auto av_j = new_data(lda*j, lclNumRows);
473 auto X_lcl_j_2d = redist_mv.getLocalViewHost(Tpetra::Access::ReadOnly);
474 auto X_lcl_j_1d = Kokkos::subview (X_lcl_j_2d, Kokkos::ALL (), j);
476 using val_type =
typename std::remove_const<typename decltype( X_lcl_j_1d )::value_type>::type;
477 Kokkos::View<val_type*, Kokkos::HostSpace> umavj ( const_cast< val_type* > ( reinterpret_cast<const val_type*> ( av_j.getRawPtr () ) ), av_j.size () );
478 Kokkos::deep_copy (umavj, X_lcl_j_1d);
485 mv_->doImport (redist_mv, *importer_, Tpetra::REPLACE);
491 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node>
492 template <
typename KV>
498 Node> >::put1dData_kokkos_view(KV& kokkos_new_data,
501 const Tpetra::Map<LocalOrdinal,
507 typedef Tpetra::Map<LocalOrdinal, GlobalOrdinal, Node> map_type;
509 TEUCHOS_TEST_FOR_EXCEPTION(
510 source_map.getRawPtr () == NULL, std::invalid_argument,
511 "Amesos2::MultiVecAdapter::put1dData_kokkos_view: source_map argument is null.");
512 TEUCHOS_TEST_FOR_EXCEPTION(
513 mv_.is_null (), std::logic_error,
514 "Amesos2::MultiVecAdapter::put1dData_kokkos_view: the internal MultiVector mv_ is null.");
516 TEUCHOS_TEST_FOR_EXCEPTION(
517 this->getMap ().is_null (), std::logic_error,
518 "Amesos2::MultiVecAdapter::put1dData_kokkos_view: this->getMap() returns null.");
520 const size_t num_vecs = getGlobalNumVectors ();
523 if ( num_vecs == 1 && this->getComm()->getRank() == 0 && this->getComm()->getSize() == 1 ) {
528 auto mv_view_to_modify_2d = mv_->getLocalViewDevice(Tpetra::Access::OverwriteAll);
530 deep_copy_only(mv_view_to_modify_2d, kokkos_new_data);
537 RCP<const map_type> srcMap;
538 if (importer_.is_null () ||
539 ! importer_->getSourceMap ()->isSameAs (* source_map) ||
540 ! importer_->getTargetMap ()->isSameAs (* (this->getMap ()))) {
546 srcMap = rcp(
new map_type(*source_map));
547 importer_ = rcp (
new import_type (srcMap, this->getMap ()));
550 srcMap = importer_->getSourceMap ();
553 if ( distribution != CONTIGUOUS_AND_ROOTED ) {
558 typedef typename multivec_t::dual_view_type::t_host::value_type tpetra_mv_view_type;
559 Kokkos::View<tpetra_mv_view_type**,
typename KV::array_layout,
560 Kokkos::HostSpace> convert_kokkos_new_data;
561 deep_copy_or_assign_view(convert_kokkos_new_data, kokkos_new_data);
562 #ifdef HAVE_TEUCHOS_COMPLEX 564 auto pData =
reinterpret_cast<Scalar*
>(convert_kokkos_new_data.data());
566 auto pData = convert_kokkos_new_data.data();
569 const multivec_t source_mv (srcMap, Teuchos::ArrayView<const scalar_t>(
570 pData, kokkos_new_data.size()), lda, num_vecs);
571 mv_->doImport (source_mv, *importer_, Tpetra::REPLACE);
574 multivec_t redist_mv (srcMap, num_vecs);
579 auto host_kokkos_new_data = Kokkos::create_mirror_view(kokkos_new_data);
580 Kokkos::deep_copy(host_kokkos_new_data, kokkos_new_data);
581 if ( redist_mv.isConstantStride() ) {
582 auto contig_local_view_2d = redist_mv.getLocalViewHost(Tpetra::Access::OverwriteAll);
583 for (
size_t j = 0; j < num_vecs; ++j) {
584 auto av_j = Kokkos::subview(host_kokkos_new_data, Kokkos::ALL, j);
585 for (
size_t i = 0; i < lda; ++i ) {
586 contig_local_view_2d(i,j) = av_j(i);
591 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::runtime_error,
"Kokkos adapter " 592 "CONTIGUOUS_AND_ROOTED not implemented for put1dData_kokkos_view " 593 "with non constant stride.");
599 mv_->doImport (redist_mv, *importer_, Tpetra::REPLACE);
605 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
611 Node> >::description()
const 613 std::ostringstream oss;
614 oss <<
"Amesos2 adapter wrapping: ";
615 oss << mv_->description();
620 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
626 Node> >::describe (Teuchos::FancyOStream& os,
627 const Teuchos::EVerbosityLevel verbLevel)
const 629 mv_->describe (os, verbLevel);
633 template <
typename Scalar,
typename LocalOrdinal,
typename GlobalOrdinal,
class Node >
634 const char* MultiVecAdapter<
638 Node> >::name =
"Amesos2 adapter for Tpetra::MultiVector";
642 #endif // AMESOS2_TPETRA_MULTIVEC_ADAPTER_DEF_HPP Copy or assign views based on memory spaces.
Definition: Amesos2_AbstractConcreteMatrixAdapter.hpp:48
Amesos2::MultiVecAdapter specialization for the Tpetra::MultiVector class.
EDistribution
Definition: Amesos2_TypeDecl.hpp:123