20 #include <stk_util/parallel/ParallelComm.hpp> 21 #include <stk_util/parallel/ParallelReduce.hpp> 23 #include <stk_mesh/base/Ghosting.hpp> 24 #include <stk_mesh/base/BulkData.hpp> 25 #include <stk_mesh/base/MetaData.hpp> 26 #include <stk_mesh/base/FieldData.hpp> 27 #include <stk_mesh/base/EntityComm.hpp> 28 #include <stk_mesh/base/Comm.hpp> 29 #include <stk_mesh/base/Trace.hpp> 40 Trace_(
"stk_classic::mesh::BulkData::create_ghosting");
42 require_ok_to_modify();
49 if ( bc.parallel_rank() == 0 ) {
50 bc.send_buffer().skip<
char>( name.size() + 1 );
55 if ( bc.parallel_rank() == 0 ) {
56 bc.send_buffer().pack<
char>( name.c_str() , name.size() + 1 );
61 const char *
const bc_name =
62 reinterpret_cast<const char *
>( bc.recv_buffer().buffer() );
64 int error = 0 != strcmp( bc_name , name.c_str() );
66 all_reduce(
parallel() , ReduceMax<1>( & error ) );
68 ThrowErrorMsgIf( error,
"Parallel name inconsistency");
72 new Ghosting( *
this , name , m_ghosting.size() , m_sync_count );
74 m_ghosting.push_back( g );
84 void insert_transitive_closure( std::set<EntityProc,EntityLess> & new_send ,
87 void comm_recv_to_send(
89 const std::set< Entity * , EntityLess > & new_recv ,
90 std::set< EntityProc , EntityLess > & new_send );
92 void comm_sync_send_recv(
94 std::set< EntityProc , EntityLess > & new_send ,
95 std::set< Entity * , EntityLess > & new_recv );
104 Trace_(
"stk_classic::mesh::BulkData::destroy_all_ghosting");
106 require_ok_to_modify();
110 for ( std::vector<Ghosting*>::iterator
111 ig = m_ghosting.begin() ; ig != m_ghosting.end() ; ++ig ) {
113 gh.m_sync_count = m_sync_count ;
118 std::vector<Entity*>::iterator ie = m_entity_comm.end();
120 while ( ie != m_entity_comm.begin() ) {
124 if ( in_receive_ghost( *entity ) ) {
125 m_entity_comm_map.comm_clear(entity->key());
130 m_entity_comm_map.comm_clear_ghosting(entity->key());
131 if ( m_entity_comm_map.comm(entity->key()).empty() ) {
138 m_entity_comm.end() , (
Entity*) NULL );
140 m_entity_comm.erase( ie , m_entity_comm.end() );
147 const std::vector<EntityProc> & add_send ,
148 const std::vector<Entity*> & remove_receive )
150 Trace_(
"stk_classic::mesh::BulkData::change_ghosting");
155 require_ok_to_modify();
157 const bool ok_mesh = & BulkData::get(ghosts) == this ;
158 const bool ok_ghost = 1 < ghosts.
ordinal();
160 bool ok_remove = true ;
164 for ( std::vector<EntityProc>::const_iterator
165 i = add_send.begin() ; ok_add && i != add_send.end() ; ++i ) {
171 for ( std::vector<Entity*>::const_iterator
172 i = remove_receive.begin() ;
173 ok_remove && i != remove_receive.end() ; ++i ) {
174 ok_remove = in_receive_ghost( ghosts , **i );
177 int ok = ok_mesh && ok_ghost && ok_add && ok_remove ;
179 all_reduce(
parallel() , ReduceMin<1>( & ok ) );
182 std::ostringstream msg ;
183 msg <<
"For ghosts " << ghosts.
name() <<
", " ;
184 if ( ! ok_mesh ) { msg <<
" : Mesh does not own this ghosting" ; }
185 if ( ! ok_ghost ) { msg <<
" : Cannot modify this ghosting" ; }
187 msg <<
" : Not owned add {" ;
188 for ( std::vector<EntityProc>::const_iterator
189 i = add_send.begin() ; i != add_send.end() ; ++i ) {
191 msg <<
" " << print_entity_key( i->first );
197 msg <<
" : Not in ghost receive {" ;
198 for ( std::vector<Entity*>::const_iterator
199 i = remove_receive.begin() ; i != remove_receive.end() ; ++i ) {
200 if ( ! in_receive_ghost( ghosts , **i ) ) {
201 msg <<
" " << print_entity_key( *i );
206 ThrowErrorMsg( msg.str() );
211 internal_change_ghosting( ghosts , add_send , remove_receive );
216 void BulkData::internal_change_ghosting(
218 const std::vector<EntityProc> & add_send ,
219 const std::vector<Entity*> & remove_receive )
221 Trace_(
"stk_classic::mesh::BulkData::internal_change_ghosting");
223 const MetaData & meta = m_mesh_meta_data ;
224 const unsigned rank_count = meta.entity_rank_count();
225 const unsigned p_size = m_parallel_size ;
231 std::set< EntityProc , EntityLess > new_send ;
232 std::set< Entity * , EntityLess > new_recv ;
245 for ( std::vector<Entity*>::const_iterator
247 Entity *
const entity = *i ;
248 if ( in_receive_ghost( ghosts , *entity ) ) {
249 new_recv.insert( entity );
255 for ( std::vector< Entity * >::const_iterator
256 i = remove_receive.begin() ; i != remove_receive.end() ; ++i ) {
257 new_recv.erase( *i );
267 for ( std::set< Entity * , EntityLess >::reverse_iterator
268 i = new_recv.rbegin() ; i != new_recv.rend() ; ++i) {
269 const unsigned erank = (*i)->entity_rank();
271 for ( PairIterRelation
272 irel = (*i)->relations(); ! irel.empty() ; ++irel ) {
273 if ( irel->entity_rank() < erank &&
274 in_receive_ghost( ghosts , * irel->entity() ) ) {
275 new_recv.insert( irel->entity() );
282 comm_recv_to_send( *
this , new_recv , new_send );
287 for ( std::vector< EntityProc >::const_iterator
288 i = add_send.begin() ; i != add_send.end() ; ++i ) {
289 insert_transitive_closure( new_send , *i );
297 comm_sync_send_recv( *
this , new_send , new_recv );
305 bool removed = false ;
307 for ( std::vector<Entity*>::reverse_iterator
308 i = m_entity_comm.rbegin() ; i != m_entity_comm.rend() ; ++i) {
309 Entity * entity = *i ;
311 const bool is_owner = entity->owner_rank() == m_parallel_rank ;
312 const bool remove_recv = ( ! is_owner ) &&
313 0 == new_recv.count( entity );
319 std::vector<EntityCommInfo> comm_ghost ;
321 comm_ghost.assign( ec.first , ec.second );
323 for ( ; ! comm_ghost.empty() ; comm_ghost.pop_back() ) {
324 const EntityCommInfo tmp = comm_ghost.back();
326 if ( 0 == new_send.count(
EntityProc( entity , tmp.proc ) ) ) {
327 m_entity_comm_map.erase(entity->key(),tmp);
331 else if ( remove_recv ) {
332 m_entity_comm_map.erase(entity->key(),ghosts);
335 if ( m_entity_comm_map.comm(entity->key()).empty() ) {
340 " FAILED attempt to destroy entity: " << print_entity_key(entity) );
346 std::vector<Entity*>::iterator i =
348 m_entity_comm.end() , (Entity*) NULL );
349 m_entity_comm.erase( i , m_entity_comm.end() );
361 const size_t entity_comm_size = m_entity_comm.size();
363 CommAll comm( m_parallel_machine );
365 for ( std::set< EntityProc , EntityLess >::iterator
366 j = new_send.begin(); j != new_send.end() ; ++j ) {
368 Entity & entity = * j->first ;
369 const unsigned int proc = j->second ;
371 if ( ! in_ghost( ghosts , entity , proc ) ) {
373 CommBuffer & buf = comm.send_buffer( proc );
374 buf.pack<
unsigned>( entity.entity_rank() );
375 pack_entity_info( buf , entity );
376 pack_field_values( buf , entity );
380 comm.allocate_buffers( p_size / 4 );
382 for ( std::set< EntityProc , EntityLess >::iterator
383 j = new_send.begin(); j != new_send.end() ; ++j ) {
385 Entity & entity = * j->first ;
386 const unsigned int proc = j->second ;
388 if ( ! in_ghost( ghosts , entity , proc ) ) {
390 CommBuffer & buf = comm.send_buffer( proc );
391 buf.pack<
unsigned>( entity.entity_rank() );
392 pack_entity_info( buf , entity );
393 pack_field_values( buf , entity );
395 m_entity_comm_map.insert(entity.key(), EntityCommInfo(ghosts.
ordinal(), proc));
397 m_entity_comm.push_back( & entity );
403 std::ostringstream error_msg ;
404 int error_count = 0 ;
406 for (
unsigned rank = 0 ; rank < rank_count ; ++rank ) {
407 for (
unsigned p = 0 ; p < p_size ; ++p ) {
408 CommBuffer & buf = comm.recv_buffer(p);
409 while ( buf.remaining() ) {
414 unsigned this_rank = ~0u ;
415 buf.peek<
unsigned>( this_rank );
417 if ( this_rank != rank ) break ;
419 buf.unpack<
unsigned>( this_rank );
423 std::vector<Relation> relations ;
425 unsigned owner = ~0u ;
427 unpack_entity_info( buf, *
this, key, owner, parts, relations );
434 std::pair<Entity*,bool> result =
435 m_entity_repo.internal_create_entity( key );
437 Entity* entity = result.first;
438 const bool created = result.second ;
439 const bool recreated = EntityLogDeleted == entity->log_query();
441 if ( created || recreated ) {
442 m_entity_repo.log_created_parallel_copy( *(entity) );
443 m_entity_repo.set_entity_owner_rank( *(entity), owner);
446 require_entity_owner( * entity , owner );
448 internal_change_entity_parts( * entity , parts ,
PartVector() );
452 if ( ! unpack_field_values( buf , * entity , error_msg ) ) {
456 const EntityCommInfo tmp( ghosts.
ordinal() , owner );
458 if ( m_entity_comm_map.insert(entity->key(),tmp) ) {
459 m_entity_comm.push_back( entity );
466 all_reduce( m_parallel_machine , ReduceSum<1>( & error_count ) );
469 ThrowErrorMsgIf( error_count, error_msg.str() );
471 if ( entity_comm_size < m_entity_comm.size() ) {
475 std::vector<Entity*>::iterator i = m_entity_comm.begin();
476 i += entity_comm_size ;
477 std::sort( i , m_entity_comm.end() , EntityLess() );
478 std::inplace_merge( m_entity_comm.begin() , i ,
479 m_entity_comm.end() , EntityLess() );
480 m_entity_comm.erase( std::unique( m_entity_comm.begin() , m_entity_comm.end() ) ,
481 m_entity_comm.end() );
485 ghosts.m_sync_count = m_sync_count ;
492 void insert_transitive_closure( std::set<EntityProc,EntityLess> & new_send ,
498 if ( entry.second != entry.first->owner_rank() &&
499 ! in_shared( * entry.first , entry.second ) ) {
501 std::pair< std::set<EntityProc,EntityLess>::iterator ,
bool >
502 result = new_send.insert( entry );
504 if ( result.second ) {
507 const unsigned etype = entry.first->entity_rank();
508 PairIterRelation irel = entry.first->relations();
510 for ( ; ! irel.empty() ; ++irel ) {
511 if ( irel->entity_rank() < etype ) {
512 EntityProc tmp( irel->entity() , entry.second );
513 insert_transitive_closure( new_send , tmp );
522 void comm_recv_to_send(
524 const std::set< Entity * , EntityLess > & new_recv ,
525 std::set< EntityProc , EntityLess > & new_send )
529 CommAll all( mesh.parallel() );
531 for (
int phase = 0; phase < 2; ++phase) {
532 for ( std::set< Entity * , EntityLess >::const_iterator
533 i = new_recv.begin() ; i != new_recv.end() ; ++i ) {
534 const unsigned owner = (*i)->owner_rank();
535 const EntityKey key = (*i)->key();
536 all.send_buffer( owner ).pack<EntityKey>( key );
546 for (
unsigned proc_rank = 0 ; proc_rank <
parallel_size ; ++proc_rank ) {
547 CommBuffer & buf = all.recv_buffer(proc_rank);
548 while ( buf.remaining() ) {
550 buf.unpack<EntityKey>( key );
551 EntityProc tmp( mesh.get_entity( key ) , proc_rank );
552 new_send.insert( tmp );
559 void comm_sync_send_recv(
561 std::set< EntityProc , EntityLess > & new_send ,
562 std::set< Entity * , EntityLess > & new_recv )
567 CommAll all( mesh.parallel() );
571 for ( std::set< EntityProc , EntityLess >::iterator
572 i = new_send.begin() ; i != new_send.end() ; ++i ) {
573 const unsigned owner = i->first->owner_rank();
574 all.send_buffer( i->second ).skip<EntityKey>(2);
576 all.send_buffer( owner ).skip<EntityKey>(2);
583 for ( std::set< EntityProc , EntityLess >::iterator
584 i = new_send.begin() ; i != new_send.end() ; ) {
585 const unsigned owner = i->first->owner_rank();
593 const EntityKey &entity_key = i->first->key();
594 const uint64_t &proc = i->second;
596 all.send_buffer( i->second ).pack(entity_key).pack(proc);
601 all.send_buffer( owner ).pack(entity_key).pack(proc);
605 std::set< EntityProc , EntityLess >::iterator jrem = i ; ++i ;
606 new_send.erase( jrem );
617 CommBuffer & buf = all.recv_buffer(p);
618 while ( buf.remaining() ) {
620 EntityKey entity_key;
623 buf.unpack(entity_key).unpack(proc);
625 Entity *
const e = mesh.get_entity( entity_key );
630 ThrowRequireMsg( e != NULL,
631 "Unknown entity key: " <<
632 MetaData::get(mesh).entity_rank_name(entity_key.rank()) <<
633 "[" << entity_key.id() <<
"]");
635 new_send.insert( tmp );
637 else if ( e != NULL ) {
642 new_recv.insert( e );
648 void insert_upward_relations(Entity& rel_entity,
649 const EntityRank rank_of_orig_entity,
650 const unsigned my_rank,
651 const unsigned share_proc,
652 std::vector<EntityProc>& send)
656 if ( rank_of_orig_entity < rel_entity.entity_rank() &&
657 rel_entity.owner_rank() == my_rank &&
658 ! in_shared( rel_entity , share_proc ) ) {
661 send.push_back( entry );
664 for ( PairIterRelation rel = rel_entity.relations() ; ! rel.empty() ; ++rel ) {
665 Entity *
const rel_of_rel_entity = rel->entity();
666 insert_upward_relations(*rel_of_rel_entity, rel_entity.entity_rank(), my_rank, share_proc, send);
676 void BulkData::internal_regenerate_shared_aura()
678 Trace_(
"stk_classic::mesh::BulkData::internal_regenerate_shared_aura");
680 require_ok_to_modify();
682 std::vector<EntityProc> send ;
687 for ( std::vector<Entity*>::const_iterator
690 Entity & entity = **i ;
692 const unsigned erank = entity.entity_rank();
696 for (
size_t j = 0 ; j < sharing.size() ; ++j ) {
698 const unsigned share_proc = sharing[j].proc ;
700 for ( PairIterRelation rel = entity.relations() ; ! rel.empty() ; ++rel ) {
702 Entity *
const rel_entity = rel->entity();
704 insert_upward_relations(*rel_entity, erank, m_parallel_rank, share_proc, send);
711 internal_change_ghosting(
shared_aura() , send , m_entity_comm );
void declare_relation(Entity &e_from, Entity &e_to, const RelationIdentifier local_id)
Declare a relation and its converse between entities in the same mesh.
Ghosting & create_ghosting(const std::string &name)
Asymmetric parallel relations for owner-to-ghosted mesh entities.
const std::vector< Entity * > & entity_comm() const
All entities with communication information.
Data for ghosting mesh entities.
void remove(PartVector &v, Part &part)
Remove a part from a properly ordered collection of parts.
void change_ghosting(Ghosting &ghosts, const std::vector< EntityProc > &add_send, const std::vector< Entity *> &remove_receive)
Change the members of a ghosting list on the sending processor.
Ghosting & shared_aura() const
Query the shared-entity aura. Is likely to be stale if ownership or sharing has changed and the 'modi...
std::pair< Entity *, unsigned > EntityProc
Pairing of an entity with a processor rank.
int parallel_rank()
function parallel_rank returns the rank of this processor in the current mpi communicator.
void destroy_all_ghosting()
Empty every single Ghosting. Same result, but more efficient than, calling change_ghosting to remove ...
ParallelMachine parallel() const
The parallel machine.
unsigned parallel_size() const
Size of the parallel machine.
Manager for an integrated collection of entities, entity relations, and buckets of field data...
A fundamental unit within the discretization of a problem domain, including but not limited to nodes...
const std::string & name() const
Text name for printing purposes only.
unsigned ordinal() const
Ordinal to identify the ghosting subset.
int parallel_size()
function parallel_size returns the number of processors in the current mpi communicator.
unsigned parallel_rank() const
Rank of the parallel machine's local processor.
std::vector< Part *> PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
PairIter< std::vector< EntityCommInfo >::const_iterator > PairIterEntityComm
Span of ( communication-subset-ordinal , process-rank ) pairs for the communication of an entity...
bool destroy_entity(Entity *&entity)
Request the destruction an entity on the local process.