dune-fem 2.8-git
mpimanager.hh
Go to the documentation of this file.
1#ifndef DUNE_FEM_MPIMANAGER_HH
2#define DUNE_FEM_MPIMANAGER_HH
3
4#include <memory>
5
6#include <dune/common/parallel/mpicommunication.hh>
7#include <dune/common/parallel/mpihelper.hh>
8
10
11#if HAVE_PETSC
13#endif
14
16
17namespace Dune
18{
19
20 namespace Fem
21 {
22
24 {
25 typedef Dune::CollectiveCommunication< MPIHelper::MPICommunicator >
27 private:
28 static MPIManager &instance ()
29 {
31 }
32
33 static bool mpiFinalized ()
34 {
35 bool finalized = false ;
36#if HAVE_MPI
37 // check that MPI was not already finalized
38 {
39 int wasFinalized = -1;
40 MPI_Finalized( &wasFinalized );
41 finalized = bool( wasFinalized );
42 }
43#endif // #if HAVE_MPI
44 return finalized ;
45 }
46
47#if HAVE_PETSC
48 struct PETSc
49 {
50 ~PETSc()
51 {
52 if( ! mpiFinalized() )
53 {
54 ::Dune::Petsc::finalize();
55 }
56 }
57
58 static void initialize( const bool verbose, int &argc, char **&argv )
59 {
60 // needed for later calling Petsc::finalize to the right time
62 ::Dune::Petsc::initialize( verbose, argc, argv );
63 }
64 };
65#endif // #if HAVE_PETSC
66
67 public:
69 {
70#if HAVE_MPI
71 // if MPI_Init was called here and finalize has not been
72 // called yet, then this is the place to call it
73 if( wasInitializedHere_ && !mpiFinalized() )
74 {
75 MPI_Finalize();
76 }
77#endif
78 }
79
80 static void initialize ( int &argc, char **&argv )
81 {
82 MPIHelper *&helper = instance().helper_;
83 std::unique_ptr< CollectiveCommunication > &comm = instance().comm_;
84
85 // the following initialization overrides the MPI_Init in dune-common
86 // to avoid a call to MPI_Finalize before all singletons have been deleted
87#if HAVE_MPI
88 int wasInitialized = -1;
89 MPI_Initialized( &wasInitialized );
90 if(!wasInitialized)
91 {
92#ifndef USE_SMP_PARALLEL
93 // standard MPI_Init
94 // call normal MPI_Init here to prevent MPIHelper to interfering
95 // with MPI_Finalize one program exit which would cause failure
96 {
97 int is_initialized = MPI_Init(&argc, &argv);
98 if( is_initialized != MPI_SUCCESS )
99 DUNE_THROW(InvalidStateException,"MPI_Init failed!");
100 }
101#else // threaded init
102 {
103 int provided;
104 // use MPI_Init_thread for hybrid parallel programs
105 int is_initialized = MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided );
106
107 if( is_initialized != MPI_SUCCESS )
108 DUNE_THROW(InvalidStateException,"MPI_Init_thread failed!");
109
110#if not defined NDEBUG && defined DUNE_DEVEL_MODE
111 // for OpenMPI provided seems to be MPI_THREAD_SINGLE
112 // but the bybrid version still works. On BlueGene systems
113 // the MPI_THREAD_FUNNELED is really needed
114 if( provided != MPI_THREAD_FUNNELED )
115 {
116 if( provided == MPI_THREAD_SINGLE )
117 dwarn << "MPI thread support = single (instead of funneled)!" << std::endl;
118 else
119 dwarn << "WARNING: MPI thread support = " << provided << " != MPI_THREAD_FUNNELED " << MPI_THREAD_FUNNELED << std::endl;
120 }
121#endif // end NDEBUG
122 }
123#endif // end USE_SMP_PARALLEL
124 instance().wasInitializedHere_ = true;
125
126 } // end if(!wasInitialized)
127#endif // end HAVE_MPI
128
129 // if already initialized, do nothing further
130 if( helper && comm )
131 return ;
132
133 // this will just initialize the static variables inside MPIHelper but
134 // not call MPI_Init again
135 helper = &MPIHelper::instance( argc, argv );
136 comm.reset( new CollectiveCommunication( helper->getCommunicator() ) );
137
138#if HAVE_PETSC
139 // initialize PETSc if pressent
140 PETSc::initialize( rank() == 0, argc, argv );
141#endif
142
143 // initialize static variables of QuadratureStorageRegistry
145 }
146
148 {
149 const std::unique_ptr< CollectiveCommunication > &comm = instance().comm_;
150 if( !comm )
151 DUNE_THROW( InvalidStateException, "MPIManager has not been initialized." );
152 return *comm;
153 }
154
155 static int rank ()
156 {
157 return comm().rank();
158 }
159
160 static int size ()
161 {
162 return comm().size();
163 }
164
165 private:
166 MPIHelper *helper_ = nullptr;
167 std::unique_ptr< CollectiveCommunication > comm_;
168 bool wasInitializedHere_ = false ;
169 };
170
171 } // namespace Fem
172
173} // namespace Dune
174
175#endif // #ifndef DUNE_FEM_MPIMANAGER_HH
Definition: bindguard.hh:11
Definition: mpimanager.hh:24
static const CollectiveCommunication & comm()
Definition: mpimanager.hh:147
Dune::CollectiveCommunication< MPIHelper::MPICommunicator > CollectiveCommunication
Definition: mpimanager.hh:26
~MPIManager()
Definition: mpimanager.hh:68
static int size()
Definition: mpimanager.hh:160
static int rank()
Definition: mpimanager.hh:155
static void initialize(int &argc, char **&argv)
Definition: mpimanager.hh:80
static void initialize()
initialize static variables
Definition: registry.hh:62
static Object & instance(Args &&... args)
return singleton instance of given Object type.
Definition: singleton.hh:101