Sierra Toolkit  Version of the Day
BulkDataOwner.cpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
13 #include <stdexcept>
14 #include <iostream>
15 #include <sstream>
16 #include <set>
17 #include <vector>
18 #include <algorithm>
19 
20 #include <stk_util/parallel/ParallelComm.hpp>
21 #include <stk_util/parallel/ParallelReduce.hpp>
22 
23 #include <stk_mesh/base/BulkData.hpp>
24 #include <stk_mesh/base/MetaData.hpp>
25 #include <stk_mesh/base/Entity.hpp>
26 #include <stk_mesh/base/EntityComm.hpp>
27 #include <stk_mesh/base/Trace.hpp>
28 
29 namespace stk_classic {
30 namespace mesh {
31 
32 //----------------------------------------------------------------------
33 
34 namespace {
35 
36 // Given an entity, if it's a ghost, insert the closure of the entity
37 // into work_list.
38 void insert_closure_ghost( Entity * const entity ,
39  const unsigned proc_local ,
40  std::set<Entity*,EntityLess> & work_list )
41 {
42  if ( ! in_owned_closure( *entity , proc_local ) ) {
43  // This entity is a ghost, put it on the work_list
44  // along with all ghosts in its closure
45 
46  std::pair< std::set<Entity*,EntityLess>::iterator , bool >
47  result = work_list.insert( entity );
48 
49  if ( result.second ) {
50  // This ghost entity is new to the list, traverse its closure.
51 
52  const unsigned entity_rank = entity->entity_rank();
53 
54  // Recurse over downward relations
55  for ( PairIterRelation
56  irel = entity->relations() ; ! irel.empty() ; ++irel ) {
57  if ( irel->entity_rank() < entity_rank ) {
58  insert_closure_ghost( irel->entity() , proc_local ,work_list );
59  }
60  }
61  }
62  }
63 }
64 
65 // Given an entity, insert the closures of every entity that has this entity
66 // in its closure. Only ghosts will be inserted.
67 void insert_transitive_ghost( Entity * const entity ,
68  const unsigned proc_local ,
69  std::set<Entity*,EntityLess> & work_list )
70 {
71  insert_closure_ghost( entity , proc_local , work_list );
72 
73  // Transitive:
74  // If this entity is a member of another entity's closure
75  // then that other entity is part of the traversal.
76 
77  const unsigned entity_rank = entity->entity_rank();
78 
79  // Recurse over upward relations
80  for ( PairIterRelation rel = entity->relations(); ! rel.empty() ; ++rel ) {
81  if ( entity_rank < rel->entity_rank() ) {
82  insert_transitive_ghost( rel->entity() , proc_local , work_list );
83  }
84  }
85 }
86 
87 //----------------------------------------------------------------------
88 
89 // Add EntityProc pairs to send_list for every entity in the closure of the
90 // entity in send_entry. All these entities will be sent to the same proc as
91 // the original send_entry.
92 void insert_closure_send(
93  const EntityProc send_entry ,
94  std::set<EntityProc,EntityLess> & send_list )
95 {
96  ThrowRequireMsg( send_entry.first->log_query() != EntityLogDeleted,
97  "Cannot send destroyed entity " << print_entity_key(send_entry.first));
98 
99  std::pair< std::set<EntityProc,EntityLess>::iterator , bool >
100  result = send_list.insert( send_entry );
101 
102  if ( result.second ) {
103  // First time this entity was inserted into the send_list.
104 
105  const unsigned erank = send_entry.first->entity_rank();
106  PairIterRelation irel = send_entry.first->relations();
107 
108  // Recurse over downward relations
109  for ( ; ! irel.empty() ; ++irel ) {
110  if ( irel->entity_rank() < erank ) {
111  const EntityProc rel_send_entry( irel->entity(), send_entry.second );
112 
113  insert_closure_send( rel_send_entry , send_list );
114  }
115  }
116  }
117 }
118 
119 //----------------------------------------------------------------------
120 
121 bool member_of_owned_closure( const Entity & e , const unsigned p_rank )
122 {
123  bool result = p_rank == e.owner_rank();
124 
125  const unsigned entity_rank = e.entity_rank();
126 
127  // Any higher ranking entities locally owned?
128  for ( PairIterRelation
129  irel = e.relations(); ! result && ! irel.empty() ; ++irel ) {
130  result = entity_rank < irel->entity_rank() &&
131  p_rank == irel->entity()->owner_rank();
132  }
133 
134  // Any higher ranking entity member of an owned closure?
135  for ( PairIterRelation
136  irel = e.relations(); ! result && ! irel.empty() ; ++irel ) {
137  result = entity_rank < irel->entity_rank() &&
138  member_of_owned_closure( * irel->entity() , p_rank );
139  }
140 
141  return result ;
142 }
143 
144 //----------------------------------------------------------------------
145 
146 // Given a vector of local ownership changes, remove duplicates and
147 // sanity check.
148 void clean_and_verify_parallel_change(
149  const BulkData & mesh ,
150  std::vector<EntityProc> & local_change )
151 {
152  const MetaData & meta = MetaData::get(mesh);
153  const unsigned p_rank = mesh.parallel_rank();
154  const unsigned p_size = mesh.parallel_size();
155  const ParallelMachine p_comm = mesh.parallel();
156 
157  size_t error_count = 0 ;
158 
159  std::ostringstream error_msg ;
160 
161  // Order and eliminate redundancies:
162  {
163  std::vector<EntityProc>::iterator i = local_change.begin() ,
164  j = local_change.end() ;
165  std::sort( i , j , EntityLess() );
166  i = std::unique( i , j );
167  local_change.erase( i , j );
168  }
169 
170  for ( std::vector<EntityProc>::iterator
171  i = local_change.begin() ; i != local_change.end() ; ++i ) {
172  std::vector<EntityProc>::iterator next = i+1 ;
173  Entity * const entity = i->first ;
174  const unsigned new_owner = i->second ;
175 
176  // Verification:
177  // 1) Cannot change the ownership of an entity that you've already marked as deleted
178  // 2) Cannot change the ownership of an entity you do not own
179  // 3) New owner must exist
180  // 4) Cannot grant ownership to two different owners
181 
182  const bool bad_null = NULL == entity ;
183 
184  // Cannot change the ownership of an entity that you've already marked as deleted
185  const bool bad_marked_deleted = ! bad_null && EntityLogDeleted == entity->log_query();
186 
187  // Cannot change the ownership of an entity you do not own
188  const bool bad_process_not_entity_owner = ! bad_null && entity->owner_rank() != p_rank ;
189 
190  // New owner must exist
191  const bool bad_new_owner_does_not_exist = p_size <= new_owner ;
192 
193  // Cannot grant ownership to two different owners
194  const bool bad_inconsistent_change = ! bad_null && next != local_change.end() && entity == next->first ;
195 
196  if ( bad_null ||
197  bad_process_not_entity_owner ||
198  bad_new_owner_does_not_exist ||
199  bad_inconsistent_change ||
200  bad_marked_deleted)
201  {
202  ++error_count ;
203 
204  error_msg << " P" << p_rank << ": " ;
205  if ( bad_null ) { error_msg << " NULL ENTITY" ; }
206  else { print_entity_key( error_msg , meta , entity->key() ); }
207  if ( bad_marked_deleted ) { error_msg << " HAS_BEEN_DELETED" ; }
208  if ( bad_process_not_entity_owner ) { error_msg << " NOT_CURRENT_OWNER" ; }
209  if ( bad_new_owner_does_not_exist ) {
210  error_msg << " BAD_NEW_OWNER( " << new_owner << " )" ;
211  }
212  if ( bad_inconsistent_change ) {
213  error_msg << " CONFLICTING_NEW_OWNER( " << new_owner ;
214  error_msg << " != " << next->second << " )" ;
215  }
216  error_msg << std::endl ;
217  }
218  else if ( new_owner == p_rank ) {
219  // Eliminate non-changes
220  i->first = NULL ;
221  i->second = 0 ;
222  }
223  }
224 
225  all_reduce( p_comm , ReduceSum<1>( & error_count ) );
226 
227  if ( error_count ) {
228  all_write_string( p_comm , std::cerr , error_msg.str() );
229 
230  ThrowErrorMsg("Bad change ownership directives\n");
231  }
232 
233  // Filter out non-changes (entity will be NULL
234  {
235  std::vector<EntityProc>::iterator i = local_change.begin(),
236  j = local_change.end();
237  i = std::remove( i , j , EntityProc(NULL,0) );
238  local_change.erase( i , j );
239  }
240 }
241 
242 //----------------------------------------------------------------------
243 // Generate a parallel consistent list of ownership changes:
244 // 1) Shared entities (not owned but in closure of an owned entity),
245 // 2) Ghosted entities (not owned and not in closure of an owned entity), and
246 // 3) Parallel index.
247 
248 void generate_parallel_change( const BulkData & mesh ,
249  const std::vector<EntityProc> & local_change ,
250  std::vector<EntityProc> & shared_change ,
251  std::vector<EntityProc> & ghosted_change )
252 {
253  const unsigned p_size = mesh.parallel_size();
254 
255  CommAll comm( mesh.parallel() );
256 
257  std::vector<unsigned> procs ;
258 
259  // pack and communicate change owner information to all
260  // processes that know about the entity
261  for ( int phase = 0; phase < 2; ++phase) {
262  for ( std::vector<EntityProc>::const_iterator
263  ip = local_change.begin() ; ip != local_change.end() ; ++ip ) {
264  Entity & entity = * ip->first ;
265  unsigned new_owner = ip->second;
266  comm_procs( entity , procs );
267  for ( std::vector<unsigned>::iterator
268  j = procs.begin() ; j != procs.end() ; ++j )
269  {
270  comm.send_buffer( *j )
271  .pack<EntityKey>( entity.key() )
272  .pack<unsigned>( new_owner );
273  }
274  }
275  if (phase == 0) { // allocation phase
276  comm.allocate_buffers( p_size / 4 , 0 );
277  }
278  else { // communication phase
279  comm.communicate();
280  }
281  }
282 
283  // unpack communicated owner information into the
284  // ghosted and shared change vectors.
285  for ( unsigned ip = 0 ; ip < p_size ; ++ip ) {
286  CommBuffer & buf = comm.recv_buffer( ip );
287  while ( buf.remaining() ) {
288  EntityProc entry ;
289  EntityKey key ;
290  buf.unpack<EntityKey>( key )
291  .unpack<unsigned>( entry.second );
292 
293  entry.first = mesh.get_entity( key );
294 
295  if ( in_receive_ghost( * entry.first ) ) {
296  ghosted_change.push_back( entry );
297  }
298  else {
299  shared_change.push_back( entry );
300  }
301  }
302  }
303 
304  std::sort( shared_change.begin() , shared_change.end() , EntityLess() );
305  std::sort( ghosted_change.begin() , ghosted_change.end() , EntityLess() );
306 }
307 
308 }
309 
310 //----------------------------------------------------------------------
311 //----------------------------------------------------------------------
312 
313 void BulkData::change_entity_owner( const std::vector<EntityProc> & arg_change )
314 {
315  Trace_("stk_classic::mesh::BulkData::change_entity_owner");
316  DiagIf(LOG_ENTITY, "arg_change: " << arg_change);
317 
318  const MetaData & meta = m_mesh_meta_data ;
319  const unsigned p_rank = m_parallel_rank ;
320  const unsigned p_size = m_parallel_size ;
321  ParallelMachine p_comm = m_parallel_machine ;
322 
323  //------------------------------
324  // Verify the input changes, generate a clean local change list, and
325  // generate the remote change list so that all processes know about
326  // pending changes.
327 
328  std::vector<EntityProc> local_change( arg_change );
329 
330  // Parallel synchronous clean up and verify the requested changes:
331  clean_and_verify_parallel_change( *this , local_change );
332 
333  //----------------------------------------
334  // Parallel synchronous determination of changing shared and ghosted.
335 
336  // The two vectors below will contain changes to ghosted and shared
337  // entities on this process coming from change-entity-owner requests
338  // on other processes.
339  std::vector<EntityProc> ghosted_change ;
340  std::vector<EntityProc> shared_change ;
341 
342  generate_parallel_change( *this , local_change ,
343  shared_change , ghosted_change );
344 
345  //------------------------------
346  // Have enough information to delete all effected ghosts.
347  // If the closure of a ghost contains a changing entity
348  // then that ghost must be deleted.
349  // Request that all ghost entities in the closure of the ghost be deleted.
350 
351  typedef std::set<EntityProc,EntityLess> EntityProcSet;
352  typedef std::set<Entity*,EntityLess> EntitySet;
353 
354  // Compute the closure of all the locally changing entities
355  EntityProcSet send_closure ;
356  for ( std::vector<EntityProc>::iterator
357  i = local_change.begin() ; i != local_change.end() ; ++i ) {
358  insert_closure_send( *i , send_closure );
359  }
360 
361  // Calculate all the ghosts that are impacted by the set of ownership
362  // changes. We look at ghosted, shared, and local changes looking for ghosts
363  // that are either in the closure of the changing entity, or have the
364  // changing entity in their closure. All modified ghosts will be removed.
365  {
366  EntitySet modified_ghosts ;
367 
368  for ( std::vector<EntityProc>::const_iterator
369  i = ghosted_change.begin() ; i != ghosted_change.end() ; ++i ) {
370  insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
371  }
372 
373  for ( std::vector<EntityProc>::const_iterator
374  i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
375  insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
376  }
377 
378  for ( EntityProcSet::iterator
379  i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
380  insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
381  }
382 
383  // The ghosted change list will become invalid
384  ghosted_change.clear();
385 
386  std::vector<EntityProc> empty_add ;
387  std::vector<Entity*> remove_modified_ghosts( modified_ghosts.begin() ,
388  modified_ghosts.end() );
389 
390  // Skip 'm_ghosting[0]' which is the shared subset.
391  for ( std::vector<Ghosting*>::iterator
392  ig = m_ghosting.begin() + 1; ig != m_ghosting.end() ; ++ig ) {
393  // parallel synchronous:
394  internal_change_ghosting( **ig , empty_add , remove_modified_ghosts );
395  }
396  }
397 
398  //------------------------------
399  // Consistently change the owner on all processes.
400  // 1) The local_change list is giving away ownership.
401  // 2) The shared_change may or may not be receiving ownership
402 
403  {
404  PartVector owned;
405  owned.push_back(& meta.locally_owned_part());
406 
407  for ( std::vector<EntityProc>::iterator
408  i = local_change.begin() ; i != local_change.end() ; ++i ) {
409  // Giving ownership, change the parts first and then
410  // the owner rank to pass the ownership test.
411  Entity* entity = i->first;
412 
413  change_entity_parts( *entity , PartVector() , owned );
414 
415  m_entity_repo.set_entity_owner_rank( *entity, i->second );
416  }
417 
418  for ( std::vector<EntityProc>::iterator
419  i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
420  Entity* entity = i->first;
421  m_entity_repo.set_entity_owner_rank( *entity, i->second);
422  if ( p_rank == i->second ) { // I receive ownership
423  change_entity_parts( *entity , owned , PartVector() );
424  }
425  }
426  }
427 
428  //------------------------------
429  // Send entities, along with their closure, to the new owner processes
430  {
431  std::ostringstream error_msg ;
432  int error_count = 0 ;
433 
434  CommAll comm( p_comm );
435 
436  for ( std::set<EntityProc,EntityLess>::iterator
437  i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
438  CommBuffer & buffer = comm.send_buffer( i->second );
439  Entity & entity = * i->first ;
440  pack_entity_info( buffer , entity );
441  pack_field_values( buffer , entity );
442  }
443 
444  comm.allocate_buffers( p_size / 4 );
445 
446  for ( std::set<EntityProc,EntityLess>::iterator
447  i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
448  CommBuffer & buffer = comm.send_buffer( i->second );
449  Entity & entity = * i->first ;
450  pack_entity_info( buffer , entity );
451  pack_field_values( buffer , entity );
452  }
453 
454  comm.communicate();
455 
456  for ( unsigned p = 0 ; p < p_size ; ++p ) {
457  CommBuffer & buf = comm.recv_buffer(p);
458  while ( buf.remaining() ) {
459  PartVector parts ;
460  std::vector<Relation> relations ;
461  EntityKey key ;
462  unsigned owner = ~0u ;
463 
464  unpack_entity_info( buf, *this, key, owner, parts, relations );
465 
466  // Received entity information will be correct,
467  // modulo the owned and shared parts
468 
469  remove( parts , meta.globally_shared_part() );
470 
471  if ( owner == p_rank ) {
472  // Must have the locally_owned_part
473  insert( parts , meta.locally_owned_part() );
474  }
475  else {
476  // Must not have the locally_owned_part
477  remove( parts , meta.locally_owned_part() );
478  }
479 
480  std::pair<Entity*,bool> result =
481  m_entity_repo.internal_create_entity( key );
482 
483  Entity* entity = result.first;
484 
485  m_entity_repo.log_created_parallel_copy( *entity );
486 
487  // The entity was copied and not created.
488 
489  m_entity_repo.set_entity_owner_rank( *entity, owner);
490 
491  internal_change_entity_parts( *entity , parts , PartVector() );
492 
493  declare_relation( *entity , relations );
494 
495  if ( ! unpack_field_values( buf , *entity , error_msg ) ) {
496  ++error_count ;
497  }
498  }
499  }
500 
501  all_reduce( p_comm , ReduceSum<1>( & error_count ) );
502  ThrowErrorMsgIf( error_count, error_msg.str() );
503 
504  // Any entity that I sent and is not in an owned closure is deleted.
505  // The owned closure will be effected by received entities, so can
506  // only clean up after the newly owned entities have been received.
507  // Destroy backwards so as not to invalidate closures in the process.
508 
509  {
510  Entity * entity = NULL ;
511 
512  for ( std::set<EntityProc,EntityLess>::iterator
513  i = send_closure.end() ; i != send_closure.begin() ; ) {
514 
515  Entity * e = (--i)->first ;
516 
517  // The same entity may be sent to more than one process.
518  // Only evaluate it once.
519 
520  if ( entity != e ) {
521  entity = e ;
522  if ( ! member_of_owned_closure( *e , p_rank ) ) {
523  ThrowRequireMsg( destroy_entity( e ),
524  "Failed to destroy entity " << print_entity_key(e) );
525  }
526  }
527  }
528  }
529 
530  send_closure.clear(); // Has been invalidated
531  }
532 }
533 
534 //----------------------------------------------------------------------
535 
536 } // namespace mesh
537 } // namespace stk_classic
538 
void declare_relation(Entity &e_from, Entity &e_to, const RelationIdentifier local_id)
Declare a relation and its converse between entities in the same mesh.
The manager of an integrated collection of parts and fields.
Definition: MetaData.hpp:56
void remove(PartVector &v, Part &part)
Remove a part from a properly ordered collection of parts.
Definition: Part.cpp:98
const EntityKey & key() const
The globally unique key ( entity type + identifier ) of this entity.
Definition: Entity.hpp:138
Part & globally_shared_part() const
Subset for the problem domain that is shared with another process. Ghost entities are not members of ...
Definition: MetaData.hpp:98
Integer type for the entity keys, which is an encoding of the entity type and entity identifier...
std::pair< Entity *, unsigned > EntityProc
Pairing of an entity with a processor rank.
Definition: Types.hpp:111
EntityModificationLog log_query() const
Query the current state of the entity log.
Definition: Entity.hpp:125
void change_entity_parts(Entity &entity, const PartVector &add_parts, const PartVector &remove_parts=PartVector())
Change the parallel-locally-owned entity&#39;s part membership by adding and/or removing parts...
Definition: BulkData.hpp:249
Part & locally_owned_part() const
Subset for the problem domain that is owned by the local process. Ghost entities are not members of t...
Definition: MetaData.hpp:93
PairIterRelation relations() const
All Entity relations for which this entity is a member. The relations are ordered from lowest entity-...
Definition: Entity.hpp:161
A fundamental unit within the discretization of a problem domain, including but not limited to nodes...
Definition: Entity.hpp:120
Sierra Toolkit.
void all_write_string(ParallelMachine arg_comm, std::ostream &arg_root_os, const std::string &arg_msg)
Write string from any or all processors to the ostream on the root processor.
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
void change_entity_owner(const std::vector< EntityProc > &arg_change)
Give away ownership of entities to other parallel processes.
EntityRank entity_rank() const
The rank of this entity.
Definition: Entity.hpp:128
std::vector< Part *> PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
Definition: Types.hpp:31
EntityRank entity_rank(const EntityKey &key)
Given an entity key, return an entity type (rank).
bool destroy_entity(Entity *&entity)
Request the destruction an entity on the local process.
Definition: BulkData.cpp:698
unsigned owner_rank() const
Parallel processor rank of the processor which owns this entity.
Definition: Entity.hpp:175
bool insert(PartVector &v, Part &part)
Insert a part into a properly ordered collection of parts. Returns true if this is a new insertion...
Definition: Part.cpp:85