Hex: src/parallel.h Source File

| Hex 2.2 Hydrogen-electron collision solver

parallel.h
Go to the documentation of this file.
1 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
2 // //
3 // / / / / __ \ \ / / //
4 // / /__ / / / _ \ \ \/ / //
5 // / ___ / | |/_/ / /\ \ //
6 // / / / / \_\ / / \ \ //
7 // //
8 // //
9 // Copyright (c) 2016, Jakub Benda, Charles University in Prague //
10 // //
11 // MIT License: //
12 // //
13 // Permission is hereby granted, free of charge, to any person obtaining a //
14 // copy of this software and associated documentation files (the "Software"), //
15 // to deal in the Software without restriction, including without limitation //
16 // the rights to use, copy, modify, merge, publish, distribute, sublicense, //
17 // and/or sell copies of the Software, and to permit persons to whom the //
18 // Software is furnished to do so, subject to the following conditions: //
19 // //
20 // The above copyright notice and this permission notice shall be included //
21 // in all copies or substantial portions of the Software. //
22 // //
23 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS //
24 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, //
25 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE //
26 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, //
27 // WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF //
28 // OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. //
29 // //
30 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
31
32 #ifndef HEX_PARALLEL
33 #define HEX_PARALLEL
34
35 #ifdef WITH_MPI
36 #include <mpi.h>
37 #else
38 #define MPI_Comm std::intptr_t
39 #endif
40
41 #include "hex-arrays.h"
42
50 class Parallel
51 {
52 public:
53
64 Parallel (int* argc, char*** argv, bool active, int groupsize)
65 : active_(active), iproc_(0), Nproc_(1), igroup_(0), Ngroup_(1), groupsize_(groupsize)
66 {
67 #ifdef WITH_MPI
68 if (active_)
69 {
70 #ifndef _OPENMP
71 // initialize MPI
72 MPI_Init(argc, argv);
73 #else
74 // initialize MPI compatible with OpenMP
75 int req_flag = MPI_THREAD_SERIALIZED, prov_flag;
76 MPI_Init_thread(argc, argv, req_flag, &prov_flag);
77
78 // check thread support
79 if (prov_flag != MPI_THREAD_SERIALIZED)
80 {
81 std::cout << "Warning: The MPI implementation doesn't support MPI_THREAD_SERIALIZED. ";
82 std::cout << "Every MPI process may thus run only on a single core." << std::endl;
83 }
84 #endif
85 // get number of processes and ID of this process
86 MPI_Comm_size(MPI_COMM_WORLD, &Nproc_);
87 MPI_Comm_rank(MPI_COMM_WORLD, &iproc_);
88
89 // check parameter compatibility
90 if (Nproc_ % groupsize_ != 0)
91 HexException("Number of processes (currently %d) must be integer mutiple of groupsize (currently %d).", Nproc_, groupsize_);
92
93 // calculate number of groups
94 Ngroup_ = Nproc_ / groupsize_;
95
96 // calculate group rank
97 igroup_ = iproc_ / groupsize_;
98
99 // prepare group communicators
100 for (int i = 0; i < Ngroup_; i++)
101 {
102 // split a new communicator for this group
103 MPI_Comm newcomm;
104 MPI_Comm_split
105 (
106 MPI_COMM_WORLD,
107 igroup_ == i ? 0 : MPI_UNDEFINED,
108 igroupproc(),
109 &newcomm
110 );
111
112 // store this communicator
113 if (igroup_ == i)
114 groupcomm_ = newcomm;
115 }
116
117 // construct also set of groups' master processes
118 MPI_Comm_split
119 (
120 MPI_COMM_WORLD,
121 IamGroupMaster() ? 0 : MPI_UNDEFINED,
122 igroup_,
123 &mastergroup_
124 );
125 }
126 #else
127 active_ = false;
128 #endif // WITH_MPI
129 }
130
131 ~Parallel ()
132 {
133 #ifdef WITH_MPI
134 if (active_)
135 MPI_Finalize();
136 #endif
137 }
138
139 //
140 // getters
141 //
142
149 bool IamMaster () const
150 {
151 return iproc_ == 0;
152 }
153
160 bool IamGroupMaster () const
161 {
162 return iproc_ % groupsize_ == 0;
163 }
164
174 bool isMyWork (int i) const
175 {
176 return i % Nproc_ == iproc_;
177 }
178
188 bool isMyGroupWork (int i) const
189 {
190 return i % Ngroup_ == igroup_;
191 }
192
194 bool active () const { return active_; }
195
197 int iproc () const { return iproc_; }
198
200 int igroup () const { return igroup_; }
201
203 int igroupproc () const { return iproc_ % groupsize_; }
204
206 int Nproc () const { return Nproc_; }
207
209 int Ngroup () const { return Ngroup_; }
210
212 int groupsize () const { return groupsize_; }
213
217 template <class T> void bcast (int owner, T* data, std::size_t N) const
218 {
219 #ifdef WITH_MPI
220 if (active_)
221 {
222 // owner : broadcast data
223 MPI_Bcast
224 (
225 data,
226 N * typeinfo<T>::ncmpt,
227 typeinfo<T>::mpicmpttype(),
228 owner,
229 MPI_COMM_WORLD
230 );
231 }
232 #endif
233 }
234
238 template <class T> void bcast (int owner, NumberArray<T> & data) const
239 {
240 #ifdef WITH_MPI
241 if (active_)
242 {
243 // owner : broadcast size
244 int size = data.size();
245 MPI_Bcast(&size, 1, MPI_INT, owner, MPI_COMM_WORLD);
246
247 // all : resize data array
248 data.resize(size);
249
250 // owner : broadcast data
251 MPI_Bcast
252 (
253 data.data(),
254 size * typeinfo<T>::ncmpt,
255 typeinfo<T>::mpicmpttype(),
256 owner,
257 MPI_COMM_WORLD
258 );
259 }
260 #endif
261 }
262
266 template <class T> void bcast_g (int igroup, int groupowner, T * data, std::size_t N) const
267 {
268 #ifdef WITH_MPI
269 if (active_ and igroup == igroup_ and groupsize_ > 1)
270 {
271 // groupowner : broadcast data
272 MPI_Bcast
273 (
274 data,
275 N * typeinfo<T>::ncmpt,
276 typeinfo<T>::mpicmpttype(),
277 groupowner,
278 groupcomm_
279 );
280 }
281 #endif
282 }
283
287 template <class T> void send (T const * array, std::size_t size, int origin, int destination) const
288 {
289 #ifdef WITH_MPI
290 if (active_ and iproc_ == origin)
291 {
292 MPI_Send
293 (
294 const_cast<T*>(array),
295 typeinfo<T>::ncmpt * size,
296 typeinfo<T>::mpicmpttype(),
297 destination,
298 origin,
299 MPI_COMM_WORLD
300 );
301 }
302 #endif
303 }
304
308 template <class T> void recv (T * array, std::size_t size, int origin, int destination) const
309 {
310 #ifdef WITH_MPI
311 if (active_ and iproc_ == destination)
312 {
313 MPI_Status status;
314
315 MPI_Recv
316 (
317 array,
318 typeinfo<T>::ncmpt * size,
319 typeinfo<T>::mpicmpttype(),
320 origin,
321 origin,
322 MPI_COMM_WORLD,
323 &status
324 );
325 }
326 #endif
327 }
328
344 template <class T> void sync (T * array, std::size_t chunksize, std::size_t Nchunk) const
345 {
346 #ifdef WITH_MPI
347 if (active_)
348 {
349 for (unsigned ichunk = 0; ichunk < Nchunk; ichunk++)
350 {
351 MPI_Bcast
352 (
353 array + ichunk * chunksize,
354 chunksize * typeinfo<T>::ncmpt,
355 typeinfo<T>::mpicmpttype(),
356 ichunk % Nproc_,
357 MPI_COMM_WORLD
358 );
359 }
360 }
361 #endif
362 }
363
364 template <class T> void sync_m (T * array, std::size_t chunksize, std::size_t Nchunk) const
365 {
366 #ifdef WITH_MPI
367 if (active_ and IamGroupMaster())
368 {
369 for (unsigned ichunk = 0; ichunk < Nchunk; ichunk++)
370 {
371 MPI_Bcast
372 (
373 array + ichunk * chunksize,
374 chunksize * typeinfo<T>::ncmpt,
375 typeinfo<T>::mpicmpttype(),
376 ichunk % Ngroup_,
377 mastergroup_
378 );
379 }
380 }
381 #endif
382 }
383
387 template <class T> void sum (T* array, std::size_t N, int owner = 0) const
388 {
389 #ifdef WITH_MPI
390 if (active_)
391 {
392 MPI_Reduce
393 (
394 (iproc_ == owner ? MPI_IN_PLACE : array),
395 (iproc_ == owner ? array : nullptr),
396 typeinfo<T>::ncmpt * N,
397 typeinfo<T>::mpicmpttype(),
398 MPI_SUM,
399 owner,
400 MPI_COMM_WORLD
401 );
402 }
403 #endif
404 }
405
415 template <class T> void syncsum (T* array, std::size_t N) const
416 {
417 #ifdef WITH_MPI
418 if (active_)
419 {
420 MPI_Allreduce
421 (
422 MPI_IN_PLACE,
423 array,
424 typeinfo<T>::ncmpt * N,
425 typeinfo<T>::mpicmpttype(),
426 MPI_SUM,
427 MPI_COMM_WORLD
428 );
429 }
430 #endif
431 }
432
442 template <class T> void syncsum_g (T* array, std::size_t N) const
443 {
444 #ifdef WITH_MPI
445 if (active_)
446 {
447 MPI_Allreduce
448 (
449 MPI_IN_PLACE,
450 array,
451 typeinfo<T>::ncmpt * N,
452 typeinfo<T>::mpicmpttype(),
453 MPI_SUM,
454 groupcomm_
455 );
456 }
457 #endif
458 }
459
463 template <class T> void sum_g (T* array, std::size_t N, int destination) const
464 {
465 #ifdef WITH_MPI
466 if (active_ and groupsize_ > 1)
467 {
468 MPI_Reduce
469 (
470 (igroupproc() == destination ? MPI_IN_PLACE : array),
471 (igroupproc() == destination ? array : nullptr),
472 typeinfo<T>::ncmpt * N,
473 typeinfo<T>::mpicmpttype(),
474 MPI_SUM,
475 destination,
476 groupcomm_
477 );
478 }
479 #endif
480 }
481
485 template <class T> void mastersum (T* array, std::size_t N, int destgroup) const
486 {
487 #ifdef WITH_MPI
488 if (active_ and IamGroupMaster() and Ngroup_ > 1)
489 {
490 MPI_Reduce
491 (
492 (igroup_ == destgroup ? MPI_IN_PLACE : array),
493 (igroup_ == destgroup ? array : nullptr),
494 typeinfo<T>::ncmpt * N,
495 typeinfo<T>::mpicmpttype(),
496 MPI_SUM,
497 destgroup,
498 mastergroup_
499 );
500 }
501 #endif
502 }
503
509 void wait () const
510 {
511 #ifdef WITH_MPI
512 if (active_)
513 MPI_Barrier(MPI_COMM_WORLD);
514 #endif
515 }
516
517 void wait_g () const
518 {
519 #ifdef WITH_MPI
520 if (active_)
521 MPI_Barrier(groupcomm_);
522 #endif
523 }
524
525 void* groupcomm () const
526 {
527 #ifdef WITH_MPI
528 if (active_)
529 return (void*)(std::intptr_t)groupcomm_;
530 #endif
531 return MPI_Comm(0);
532 }
533
534 private:
535
536 // whether the MPI is on
537 bool active_;
538
539 // global communicator rank
540 int iproc_;
541
542 // global communicator size
543 int Nproc_;
544
545 // group index
546 int igroup_;
547
548 // group count
549 int Ngroup_;
550 #ifdef WITH_MPI
551 // MPI group communicator
552 MPI_Comm groupcomm_;
553
554 // MPI master group communicator
555 MPI_Comm mastergroup_;
556 #endif
557 // local communicator size
558 int groupsize_;
559 };
560
561 #endif
Parallel::mastersum
void mastersum(T *array, std::size_t N, int destgroup) const
Sum array from all group masters to one of the group masters.
Definition: parallel.h:485
Parallel::Nproc
int Nproc() const
Returns the size of the communicator (process count).
Definition: parallel.h:206
Parallel::iproc
int iproc() const
Returns the rank of the communicator (process is).
Definition: parallel.h:197
Parallel::recv
void recv(T *array, std::size_t size, int origin, int destination) const
Receive data from a process.
Definition: parallel.h:308
Parallel::Ngroup
int Ngroup() const
Returns the number of groups.
Definition: parallel.h:209
Parallel::active
bool active() const
Returns true if the MPI is active.
Definition: parallel.h:194
Parallel::igroupproc
int igroupproc() const
Returns index of rank within the group.
Definition: parallel.h:203
Parallel::wait_g
void wait_g() const
Definition: parallel.h:517
Parallel::groupsize
int groupsize() const
Return group size.
Definition: parallel.h:212
Parallel::bcast
void bcast(int owner, T *data, std::size_t N) const
Broadcast array from owner to everyone.
Definition: parallel.h:217
Parallel::isMyWork
bool isMyWork(int i) const
Returns true if the work item is assigned to this process.
Definition: parallel.h:174
Parallel::bcast_g
void bcast_g(int igroup, int groupowner, T *data, std::size_t N) const
Broadcast array from owner to everyone in the group.
Definition: parallel.h:266
Parallel::bcast
void bcast(int owner, NumberArray< T > &data) const
Broadcast array from owner to everyone.
Definition: parallel.h:238
Parallel::sum_g
void sum_g(T *array, std::size_t N, int destination) const
Sum array within group to a given process.
Definition: parallel.h:463
Parallel::wait
void wait() const
Wait for completition of all running tasks.
Definition: parallel.h:509
Parallel::isMyGroupWork
bool isMyGroupWork(int i) const
Returns true if the work item is assigned to this process&#39; group.
Definition: parallel.h:188
Parallel::send
void send(T const *array, std::size_t size, int origin, int destination) const
Send data to a process.
Definition: parallel.h:287
Parallel::sync
void sync(T *array, std::size_t chunksize, std::size_t Nchunk) const
Synchronize across processes by composition.
Definition: parallel.h:344
Parallel::groupcomm
void * groupcomm() const
Definition: parallel.h:525
Parallel
MPI info.
Definition: parallel.h:50
Parallel::~Parallel
~Parallel()
Definition: parallel.h:131
Parallel::IamMaster
bool IamMaster() const
Returns true if this process is the master process.
Definition: parallel.h:149
Parallel::IamGroupMaster
bool IamGroupMaster() const
Returns true if this process is the master process within a group.
Definition: parallel.h:160
Parallel::Parallel
Parallel(int *argc, char ***argv, bool active, int groupsize)
Constructor.
Definition: parallel.h:64
Parallel::sum
void sum(T *array, std::size_t N, int owner=0) const
Sum arrays to node.
Definition: parallel.h:387
Parallel::syncsum_g
void syncsum_g(T *array, std::size_t N) const
Synchronize across group&#39;s processes by summing.
Definition: parallel.h:442
Parallel::igroup
int igroup() const
Returns the rank of process group.
Definition: parallel.h:200
Parallel::syncsum
void syncsum(T *array, std::size_t N) const
Synchronize across processes by summing.
Definition: parallel.h:415
Parallel::sync_m
void sync_m(T *array, std::size_t chunksize, std::size_t Nchunk) const
Definition: parallel.h:364