dune-spgrid 2.7
communication.hh
Go to the documentation of this file.
1#ifndef DUNE_SPGRID_COMMUNICATION_HH
2#define DUNE_SPGRID_COMMUNICATION_HH
3
4#include <dune/common/hybridutilities.hh>
5#include <dune/common/parallel/communication.hh>
6#include <dune/common/parallel/mpicommunication.hh>
7#include <dune/common/parallel/mpitraits.hh>
8#include <dune/common/visibility.hh>
9
10#include <dune/grid/common/exceptions.hh>
11#include <dune/grid/common/datahandleif.hh>
12
15
16namespace Dune
17{
18
19 // SPCommunicationTraits
20 // ---------------------
21
22 template< class Comm >
24 {
25 typedef Dune::CollectiveCommunication< Comm > CollectiveCommunication;
26
27 template< class C >
28 static CollectiveCommunication comm ( const C & )
29 {
30 return defaultComm();
31 }
32
34 {
36 }
37 };
38
39#if HAVE_MPI
40 template<>
41 struct SPCommunicationTraits< MPI_Comm >
42 {
43 typedef Dune::CollectiveCommunication< MPI_Comm > CollectiveCommunication;
44
45 static CollectiveCommunication comm ( const MPI_Comm &mpiComm )
46 {
47 return CollectiveCommunication( mpiComm );
48 }
49
51 {
52 return comm( MPI_COMM_WORLD );
53 }
54 };
55#endif // #if HAVE_MPI
56
57
58
59 namespace __SPGrid
60 {
61
62 DUNE_EXPORT inline int getCommTag ()
63 {
64 static unsigned char counter = 0;
65 return int( counter++ ) + 1536;
66 }
67
68 } // namespace __SPGrid
69
70
71
72 // SPCommunication
73 // ---------------
74
75 template< class Grid, class DataHandle >
77 {
78 static const int dimension = Grid::dimension;
79
82
83 typedef typename DataHandle::DataType DataType;
84
86
87 private:
90
91 public:
92 SPCommunication ( const GridLevel &gridLevel, DataHandle &dataHandle,
93 InterfaceType iftype, CommunicationDirection dir );
94
95 SPCommunication ( const SPCommunication & ) = delete;
97
99
100 bool pending () const { return bool( interface_ ); }
101
102 void wait ();
103
104 private:
105 const GridLevel &gridLevel_;
106 DataHandle &dataHandle_;
107 const Interface *interface_;
108 CommunicationDirection dir_;
109 int tag_;
110 bool fixedSize_;
111 std::vector< WriteBuffer > writeBuffers_;
112 std::vector< ReadBuffer > readBuffers_;
113 };
114
115
116
117 // Implementation of SPCommunication
118 // ---------------------------------
119
120 template< class Grid, class DataHandle >
122 ::SPCommunication ( const GridLevel &gridLevel, DataHandle &dataHandle,
123 InterfaceType iftype, CommunicationDirection dir )
124 : gridLevel_( gridLevel ),
125 dataHandle_( dataHandle ),
126 interface_( &gridLevel.commInterface( iftype ) ),
127 dir_( dir ),
128 tag_( __SPGrid::getCommTag() ),
129 fixedSize_( true )
130 {
131 for( int codim = 0; codim <= dimension; ++codim )
132 fixedSize_ &= !dataHandle_.contains( dimension, codim ) || dataHandle_.fixedsize( dimension, codim );
133
134 const std::size_t numLinks = interface_->size();
135 readBuffers_.reserve( numLinks );
136
137 if( fixedSize_ )
138 {
139 for( typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
140 {
141 readBuffers_.emplace_back( gridLevel.grid().comm() );
142 std::size_t size = 0;
143 const PartitionList &partitionList = it->receiveList( dir );
144 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [ this, &partitionList, &size ] ( auto codim ) {
146
147 if( !dataHandle_.contains( dimension, codim ) )
148 return;
149
150 const Iterator end( gridLevel_, partitionList, typename Iterator::End() );
151 for( Iterator it( gridLevel_, partitionList, typename Iterator::Begin() ); it != end; ++it )
152 size += dataHandle_.size( *it );
153 } );
154 size *= sizeof( DataType );
155 readBuffers_.back().receive( it->rank(), tag_, size );
156 }
157 }
158
159 writeBuffers_.reserve( numLinks );
160 for( typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
161 {
162 writeBuffers_.emplace_back( gridLevel.grid().comm() );
163 const PartitionList &partitionList = it->sendList( dir );
164 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [ this, &partitionList ] ( auto codim ) {
166
167 if( !dataHandle_.contains( dimension, codim ) )
168 return;
169
170 const bool fixedSize = dataHandle_.fixedsize( dimension, codim );
171 const Iterator end( gridLevel_, partitionList, typename Iterator::End() );
172 for( Iterator it( gridLevel_, partitionList, typename Iterator::Begin() ); it != end; ++it )
173 {
174 const auto &entity = *it;
175 if( !fixedSize )
176 writeBuffers_.back().write( static_cast< int >( dataHandle_.size( entity ) ) );
177#ifndef NDEBUG
178 const std::size_t posBeforeGather = writeBuffers_.back().position();
179#endif // #ifndef NDEBUG
180 dataHandle_.gather( writeBuffers_.back(), entity );
181#ifndef NDEBUG
182 const std::size_t posAfterGather = writeBuffers_.back().position();
183 const std::size_t sizeInBytes = dataHandle_.size( entity ) * sizeof( DataType );
184 if( posAfterGather - posBeforeGather != sizeInBytes )
185 DUNE_THROW( GridError, "Number of bytes written (" << (posAfterGather - posBeforeGather) << ") does not coincide with reported size (" << sizeInBytes << ")" );
186#endif // #ifndef NDEBUG
187 }
188 } );
189 writeBuffers_.back().send( it->rank(), tag_ );
190 }
191 }
192
193
194 template< class Grid, class DataHandle >
196 : gridLevel_( other.gridLevel_ ),
197 dataHandle_( other.dataHandle_ ),
198 interface_( other.interface_ ),
199 dir_( other.dir_ ),
200 tag_( other.tag_ ),
201 fixedSize_( other.fixedSize_ ),
202 writeBuffers_( std::move( other.writeBuffers_ ) ),
203 readBuffers_( std::move( other.readBuffers_ ) )
204 {
205 other.interface_ = nullptr;
206 }
207
208
209 template< class Grid, class DataHandle >
211 {
212 if( !pending() )
213 return;
214
215 const std::size_t numLinks = interface_->size();
216
217 if( !fixedSize_ )
218 {
219 for( std::size_t i = 0; i < numLinks; ++i )
220 {
221 readBuffers_.emplace_back( gridLevel_.grid().comm() );
222 readBuffers_.back().receive( tag_ );
223 }
224 }
225
226 for( std::size_t i = 0; i < numLinks; ++i )
227 {
228 const typename std::vector< ReadBuffer >::iterator buffer = waitAny( readBuffers_ );
229 for( typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
230 {
231 if( it->rank() == buffer->rank() )
232 {
233 const PartitionList &partitionList = it->receiveList( dir_ );
234 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [ this, &partitionList, buffer ] ( auto codim ) {
236
237 if( !dataHandle_.contains( dimension, codim ) )
238 return;
239
240 const bool fixedSize = dataHandle_.fixedsize( dimension, codim );
241 const Iterator end( gridLevel_, partitionList, typename Iterator::End() );
242 for( Iterator it( gridLevel_, partitionList, typename Iterator::Begin() ); it != end; ++it )
243 {
244 const auto &entity = *it;
245
246 int size;
247 if( !fixedSize )
248 buffer->read( size );
249 else
250 size = dataHandle_.size( entity );
251#ifndef NDEBUG
252 const std::size_t posBeforeGather = buffer->position();
253#endif // #ifndef NDEBUG
254 dataHandle_.scatter( *buffer, entity, size );
255#ifndef NDEBUG
256 const std::size_t posAfterGather = buffer->position();
257 const std::size_t sizeInBytes = static_cast< std::size_t >( size ) * sizeof( DataType );
258 if( posAfterGather - posBeforeGather != sizeInBytes )
259 DUNE_THROW( GridError, "Number of bytes read (" << (posAfterGather - posBeforeGather) << ") does not coincide with reported size (" << sizeInBytes << ")" );
260#endif // #ifndef NDEBUG
261 }
262 } );
263 break;
264 }
265 }
266 }
267 readBuffers_.clear();
268
269 for( typename std::vector< WriteBuffer >::iterator it = writeBuffers_.begin(); it != writeBuffers_.end(); ++it )
270 it->wait();
271 writeBuffers_.clear();
272
273 interface_ = nullptr;
274 }
275
276} // namespace Dune
277
278#endif // #ifndef DUNE_SPGRID_COMMUNICATION_HH
STL namespace.
Definition: iostream.hh:7
DUNE_EXPORT int getCommTag()
Definition: communication.hh:62
Definition: communication.hh:24
Dune::CollectiveCommunication< Comm > CollectiveCommunication
Definition: communication.hh:25
static CollectiveCommunication comm(const C &)
Definition: communication.hh:28
static CollectiveCommunication defaultComm()
Definition: communication.hh:33
static CollectiveCommunication defaultComm()
Definition: communication.hh:50
static CollectiveCommunication comm(const MPI_Comm &mpiComm)
Definition: communication.hh:45
Dune::CollectiveCommunication< MPI_Comm > CollectiveCommunication
Definition: communication.hh:43
Definition: communication.hh:77
static const int dimension
Definition: communication.hh:78
bool pending() const
Definition: communication.hh:100
DataHandle::DataType DataType
Definition: communication.hh:83
GridLevel::CommInterface Interface
Definition: communication.hh:85
SPCommunication(const SPCommunication &)=delete
void wait()
Definition: communication.hh:210
SPCommunication(const GridLevel &gridLevel, DataHandle &dataHandle, InterfaceType iftype, CommunicationDirection dir)
Definition: communication.hh:122
SPPartitionList< dimension > PartitionList
Definition: communication.hh:81
SPGridLevel< Grid > GridLevel
Definition: communication.hh:80
~SPCommunication()
Definition: communication.hh:98
Definition: gridlevel.hh:35
const Grid & grid() const
Definition: gridlevel.hh:82
Definition: iterator.hh:19
Definition: linkage.hh:54
NodeContainer::const_iterator Iterator
Definition: linkage.hh:60
Iterator end() const
Definition: linkage.hh:68
Iterator begin() const
Definition: linkage.hh:67
std::size_t size() const
Definition: linkage.hh:70
Definition: messagebuffer.hh:91
Definition: messagebuffer.hh:204
Definition: partitionlist.hh:16