MueLu  Version of the Day
MueLu_Zoltan2Interface_def.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // MueLu: A package for multigrid based preconditioning
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact
39 // Jonathan Hu (jhu@sandia.gov)
40 // Andrey Prokopenko (aprokop@sandia.gov)
41 // Ray Tuminaro (rstumin@sandia.gov)
42 //
43 // ***********************************************************************
44 //
45 // @HEADER
46 #ifndef MUELU_ZOLTAN2INTERFACE_DEF_HPP
47 #define MUELU_ZOLTAN2INTERFACE_DEF_HPP
48 
49 #include <sstream>
50 #include <set>
51 
53 #if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI)
54 
55 #include <Zoltan2_XpetraMultiVectorAdapter.hpp>
56 #include <Zoltan2_PartitioningProblem.hpp>
57 
58 #include <Teuchos_Utils.hpp>
59 #include <Teuchos_DefaultMpiComm.hpp>
60 #include <Teuchos_OpaqueWrapper.hpp>
61 
62 #include "MueLu_Level.hpp"
63 #include "MueLu_Exceptions.hpp"
64 #include "MueLu_Monitor.hpp"
65 #include "MueLu_Utilities.hpp"
66 
67 namespace MueLu {
68 
69  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
71  defaultZoltan2Params = rcp(new ParameterList());
72  defaultZoltan2Params->set("algorithm", "multijagged");
73  defaultZoltan2Params->set("partitioning_approach", "partition");
74  }
75 
76  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
78  RCP<ParameterList> validParamList = rcp(new ParameterList());
79 
80  validParamList->set< int > ("rowWeight", 0, "Default weight to rows (total weight = nnz + rowWeight");
81 
82  validParamList->set< RCP<const FactoryBase> > ("A", Teuchos::null, "Factory of the matrix A");
83  validParamList->set< RCP<const FactoryBase> > ("number of partitions", Teuchos::null, "Instance of RepartitionHeuristicFactory.");
84  validParamList->set< RCP<const FactoryBase> > ("Coordinates", Teuchos::null, "Factory of the coordinates");
85  validParamList->set< RCP<const ParameterList> > ("ParameterList", Teuchos::null, "Zoltan2 parameters");
86 
87  return validParamList;
88  }
89 
90 
91  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
93  Input(currentLevel, "A");
94  Input(currentLevel, "number of partitions");
95  Input(currentLevel, "Coordinates");
96  }
97 
98  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
100  FactoryMonitor m(*this, "Build", level);
101 
102  RCP<Matrix> A = Get<RCP<Matrix> >(level, "A");
103  RCP<const Map> rowMap = A->getRowMap();
104 
105  typedef Xpetra::MultiVector<double, LocalOrdinal, GlobalOrdinal, Node> dMultiVector;
106  RCP<dMultiVector> coords = Get<RCP<dMultiVector> >(level, "Coordinates");
107  RCP<const Map> map = coords->getMap();
108  GO numElements = map->getNodeNumElements();
109 
110  LO blkSize = A->GetFixedBlockSize();
111 
112  // Check that the number of local coordinates is consistent with the #rows in A
113  TEUCHOS_TEST_FOR_EXCEPTION(rowMap->getNodeNumElements()/blkSize != coords->getLocalLength(), Exceptions::Incompatible,
114  "Coordinate vector length (" + toString(coords->getLocalLength()) << " is incompatible with number of block rows in A ("
115  + toString(rowMap->getNodeNumElements()/blkSize) + "The vector length should be the same as the number of mesh points.");
116 #ifdef HAVE_MUELU_DEBUG
117  GO indexBase = rowMap->getIndexBase();
118  GetOStream(Runtime0) << "Checking consistence of row and coordinates maps" << std::endl;
119  // Make sure that logical blocks in row map coincide with logical nodes in coordinates map
120  ArrayView<const GO> rowElements = rowMap->getNodeElementList();
121  ArrayView<const GO> coordsElements = map ->getNodeElementList();
122  for (LO i = 0; i < Teuchos::as<LO>(numElements); i++)
123  TEUCHOS_TEST_FOR_EXCEPTION((coordsElements[i]-indexBase)*blkSize + indexBase != rowElements[i*blkSize],
124  Exceptions::RuntimeError, "i = " << i << ", coords GID = " << coordsElements[i]
125  << ", row GID = " << rowElements[i*blkSize] << ", blkSize = " << blkSize << std::endl);
126 #endif
127 
128  int numParts = Get<int>(level, "number of partitions");
129  if (numParts == 1) {
130  // Single processor, decomposition is trivial: all zeros
131  RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
132  Set(level, "Partition", decomposition);
133  return;
134  } else if (numParts == -1) {
135  // No repartitioning
136  RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Teuchos::null; //Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
137  //decomposition->putScalar(Teuchos::as<Scalar>(rowMap->getComm()->getRank()));
138  Set(level, "Partition", decomposition);
139  return;
140  }
141 
142 
143  const ParameterList& pL = GetParameterList();
144 
145  RCP<const ParameterList> providedList = pL.get<RCP<const ParameterList> >("ParameterList");
146  ParameterList Zoltan2Params;
147  if (providedList != Teuchos::null)
148  Zoltan2Params = *providedList;
149 
150  // Merge defalt Zoltan2 parameters with user provided
151  // If default and user parameters contain the same parameter name, user one is always preferred
152  for (ParameterList::ConstIterator param = defaultZoltan2Params->begin(); param != defaultZoltan2Params->end(); param++) {
153  const std::string& pName = defaultZoltan2Params->name(param);
154  if (!Zoltan2Params.isParameter(pName))
155  Zoltan2Params.set(pName, defaultZoltan2Params->get<std::string>(pName));
156  }
157  Zoltan2Params.set("num_global_parts", Teuchos::as<int>(numParts));
158 
159  GetOStream(Runtime0) << "Zoltan2 parameters:\n----------\n" << Zoltan2Params << "----------" << std::endl;
160 
161  const std::string& algo = Zoltan2Params.get<std::string>("algorithm");
162  TEUCHOS_TEST_FOR_EXCEPTION(algo != "multijagged" && algo != "rcb", Exceptions::RuntimeError,
163  "Unknown partitioning algorithm: \"" << algo << "\"");
164 
165  typedef Zoltan2::XpetraMultiVectorAdapter<dMultiVector> InputAdapterType;
166  typedef Zoltan2::PartitioningProblem<InputAdapterType> ProblemType;
167 
168  int rowWeight = pL.get<int>("rowWeight");
169  GetOStream(Runtime0) << "Using weights formula: nnz + " << rowWeight << std::endl;
170 
171  Array<double> weightsPerRow(numElements);
172  for (LO i = 0; i < numElements; i++) {
173  weightsPerRow[i] = 0.0;
174 
175  for (LO j = 0; j < blkSize; j++) {
176  weightsPerRow[i] += A->getNumEntriesInLocalRow(i*blkSize+j);
177  // Zoltan2 pqJagged gets as good partitioning as Zoltan RCB in terms of nnz
178  // but Zoltan also gets a good partioning in rows, which sometimes does not
179  // happen for Zoltan2. So here is an attempt to get a better row partitioning
180  // without significantly screwing up nnz partitioning
181  // NOTE: no good heuristic here, the value was chosen almost randomly
182  weightsPerRow[i] += rowWeight;
183  }
184  }
185 
186  std::vector<int> strides;
187  std::vector<const double*> weights(1, weightsPerRow.getRawPtr());
188 
189  RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
190  RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
191 
192  InputAdapterType adapter(coords, weights, strides);
193  RCP<ProblemType> problem(new ProblemType(&adapter, &Zoltan2Params, (*zoltanComm)()));
194 
195  {
196  SubFactoryMonitor m1(*this, "Zoltan2 " + toString(algo), level);
197  problem->solve();
198  }
199 
200  RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Xpetra::VectorFactory<GO,LO,GO,NO>::Build(rowMap, false);
201  ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
202 
203  const typename InputAdapterType::part_t * parts = problem->getSolution().getPartListView();
204 
205  for (GO i = 0; i < numElements; i++) {
206  int partNum = parts[i];
207 
208  for (LO j = 0; j < blkSize; j++)
209  decompEntries[i*blkSize + j] = partNum;
210  }
211 
212  Set(level, "Partition", decomposition);
213  }
214 
215 } //namespace MueLu
216 
217 #endif //if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI)
218 
219 #endif // MUELU_ZOLTAN2INTERFACE_DEF_HPP
void DeclareInput(Level &currentLevel) const
Specifies the data that this class needs, and the factories that generate that data.
std::string toString(const T &what)
Little helper function to convert non-string types to strings.
Timer to be used in factories. Similar to Monitor but with additional timers.
One-liner description of what is happening.
Namespace for MueLu classes and methods.
Exception throws to report incompatible objects (like maps).
void Build(Level &currentLevel) const
Build an object with this factory.
Class that holds all level-specific information.
Definition: MueLu_Level.hpp:99
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
Exception throws to report errors in the internal logical of the program.
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.