parallel.h

Go to the documentation of this file.

//  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *  //
//                                                                                   //
//                       / /   / /    __    \ \  / /                                 //
//                      / /__ / /   / _ \    \ \/ /                                  //
//                     /  ___  /   | |/_/    / /\ \                                  //
//                    / /   / /    \_\      / /  \ \                                 //
//                                                                                   //
//                                                                                   //
//  Copyright (c) 2016, Jakub Benda, Charles University in Prague                    //
//                                                                                   //
// MIT License:                                                                      //
//                                                                                   //
//  Permission is hereby granted, free of charge, to any person obtaining a          //
// copy of this software and associated documentation files (the "Software"),        //
// to deal in the Software without restriction, including without limitation         //
// the rights to use, copy, modify, merge, publish, distribute, sublicense,          //
// and/or sell copies of the Software, and to permit persons to whom the             //
// Software is furnished to do so, subject to the following conditions:              //
//                                                                                   //
//  The above copyright notice and this permission notice shall be included          //
// in all copies or substantial portions of the Software.                            //
//                                                                                   //
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS          //
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       //
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE       //
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, //
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF         //
// OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  //
//                                                                                   //
//  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *  //

#ifndef HEX_PARALLEL
#define HEX_PARALLEL

#ifdef WITH_MPI
#include <mpi.h>
#else
#define MPI_Comm std::intptr_t
#endif

#include "hex-arrays.h"

class Parallel
{
    public:
        
        Parallel (int* argc, char*** argv, bool active, int groupsize)
            : active_(active), iproc_(0), Nproc_(1), igroup_(0), Ngroup_(1), groupsize_(groupsize)
        {
#ifdef WITH_MPI
            if (active_)
            {
    #ifndef _OPENMP
                // initialize MPI
                MPI_Init(argc, argv);
    #else
                // initialize MPI compatible with OpenMP
                int req_flag = MPI_THREAD_SERIALIZED, prov_flag;
                MPI_Init_thread(argc, argv, req_flag, &prov_flag);
                
                // check thread support
                if (prov_flag != MPI_THREAD_SERIALIZED)
                {
                    std::cout << "Warning: The MPI implementation doesn't support MPI_THREAD_SERIALIZED. ";
                    std::cout << "Every MPI process may thus run only on a single core." << std::endl;
                }
    #endif
                // get number of processes and ID of this process
                MPI_Comm_size(MPI_COMM_WORLD, &Nproc_);
                MPI_Comm_rank(MPI_COMM_WORLD, &iproc_);
                
                // check parameter compatibility
                if (Nproc_ % groupsize_ != 0)
                    HexException("Number of processes (currently %d) must be integer mutiple of groupsize (currently %d).", Nproc_, groupsize_);
                
                // calculate number of groups
                Ngroup_ = Nproc_ / groupsize_;
                
                // calculate group rank
                igroup_ = iproc_ / groupsize_;
                
                // prepare group communicators
                for (int i = 0; i < Ngroup_; i++)
                {
                    // split a new communicator for this group
                    MPI_Comm newcomm;
                    MPI_Comm_split
                    (
                        MPI_COMM_WORLD,
                        igroup_ == i ? 0 : MPI_UNDEFINED,
                        igroupproc(),
                        &newcomm
                    );
                    
                    // store this communicator
                    if (igroup_ == i)
                        groupcomm_ = newcomm;
                }
                
                // construct also set of groups' master processes
                MPI_Comm_split
                (
                    MPI_COMM_WORLD,
                    IamGroupMaster() ? 0 : MPI_UNDEFINED,
                    igroup_,
                    &mastergroup_
                );
            }
#else
            active_ = false;
#endif // WITH_MPI
        }
        
        ~Parallel ()
        {
#ifdef WITH_MPI
            if (active_)
                MPI_Finalize();
#endif
        }
        
        //
        // getters
        //
        
        bool IamMaster () const
        {
            return iproc_ == 0;
        }
        
        bool IamGroupMaster () const
        {
            return iproc_ % groupsize_ == 0;
        }
        
        bool isMyWork (int i) const
        {
            return i % Nproc_ == iproc_;
        }
        
        bool isMyGroupWork (int i) const
        {
            return i % Ngroup_ == igroup_;
        }
        
        bool active () const { return active_; }
        
        int iproc () const { return iproc_; }
        
        int igroup () const { return igroup_; }
        
        int igroupproc () const { return iproc_ % groupsize_; }
        
        int Nproc () const { return Nproc_; }
        
        int Ngroup () const { return Ngroup_; }
        
        int groupsize () const { return groupsize_; }
        
        template <class T> void bcast (int owner, T* data, std::size_t N) const
        {
#ifdef WITH_MPI
            if (active_)
            {
                // owner : broadcast data
                MPI_Bcast
                (
                    data,
                    N * typeinfo<T>::ncmpt,
                    typeinfo<T>::mpicmpttype(),
                    owner,
                    MPI_COMM_WORLD
                );
            }
#endif
        }
        
        template <class T> void bcast (int owner, NumberArray<T> & data) const
        {
#ifdef WITH_MPI
            if (active_)
            {
                // owner : broadcast size
                int size = data.size();
                MPI_Bcast(&size, 1, MPI_INT, owner, MPI_COMM_WORLD);
                
                // all : resize data array
                data.resize(size);
                
                // owner : broadcast data
                MPI_Bcast
                (
                    data.data(),
                    size * typeinfo<T>::ncmpt,
                    typeinfo<T>::mpicmpttype(),
                    owner,
                    MPI_COMM_WORLD
                );
            }
#endif
        }
        
        template <class T> void bcast_g (int igroup, int groupowner, T * data, std::size_t N) const
        {
#ifdef WITH_MPI
            if (active_ and igroup == igroup_ and groupsize_ > 1)
            {
                // groupowner : broadcast data
                MPI_Bcast
                (
                    data,
                    N * typeinfo<T>::ncmpt,
                    typeinfo<T>::mpicmpttype(),
                    groupowner,
                    groupcomm_
                );
            }
#endif
        }
        
        template <class T> void send (T const * array, std::size_t size, int origin, int destination) const
        {
#ifdef WITH_MPI
            if (active_ and iproc_ == origin)
            {
                MPI_Send
                (
                    const_cast<T*>(array),
                    typeinfo<T>::ncmpt * size,
                    typeinfo<T>::mpicmpttype(),
                    destination,
                    origin,
                    MPI_COMM_WORLD
                );
            }
#endif
        }
        
        template <class T> void recv (T * array, std::size_t size, int origin, int destination) const
        {
#ifdef WITH_MPI
            if (active_ and iproc_ == destination)
            {
                MPI_Status status;
                
                MPI_Recv
                (
                    array,
                    typeinfo<T>::ncmpt * size,
                    typeinfo<T>::mpicmpttype(),
                    origin,
                    origin,
                    MPI_COMM_WORLD,
                    &status
                );
            }
#endif
        }
        
        template <class T> void sync (T * array, std::size_t chunksize, std::size_t Nchunk) const
        {
#ifdef WITH_MPI
            if (active_)
            {
                for (unsigned ichunk = 0; ichunk < Nchunk; ichunk++)
                {
                    MPI_Bcast
                    (
                        array + ichunk * chunksize,
                        chunksize * typeinfo<T>::ncmpt,
                        typeinfo<T>::mpicmpttype(),
                        ichunk % Nproc_,
                        MPI_COMM_WORLD
                    );
                }
            }
#endif
        }
        
        template <class T> void sync_m (T * array, std::size_t chunksize, std::size_t Nchunk) const
        {
#ifdef WITH_MPI
            if (active_ and IamGroupMaster())
            {
                for (unsigned ichunk = 0; ichunk < Nchunk; ichunk++)
                {
                    MPI_Bcast
                    (
                        array + ichunk * chunksize,
                        chunksize * typeinfo<T>::ncmpt,
                        typeinfo<T>::mpicmpttype(),
                        ichunk % Ngroup_,
                        mastergroup_
                    );
                }
            }
#endif
        }
        
        template <class T> void sum (T* array, std::size_t N, int owner = 0) const
        {
#ifdef WITH_MPI
            if (active_)
            {
                MPI_Reduce
                (
                    (iproc_ == owner ? MPI_IN_PLACE : array),
                    (iproc_ == owner ? array : nullptr),
                    typeinfo<T>::ncmpt * N,
                    typeinfo<T>::mpicmpttype(),
                    MPI_SUM,
                    owner,
                    MPI_COMM_WORLD
                );
            }
#endif
        }
        
        template <class T> void syncsum (T* array, std::size_t N) const
        {
#ifdef WITH_MPI
            if (active_)
            {
                MPI_Allreduce
                (
                    MPI_IN_PLACE,
                    array,
                    typeinfo<T>::ncmpt * N,
                    typeinfo<T>::mpicmpttype(),
                    MPI_SUM,
                    MPI_COMM_WORLD
                );
            }
#endif
        }
        
        template <class T> void syncsum_g (T* array, std::size_t N) const
        {
#ifdef WITH_MPI
            if (active_)
            {
                MPI_Allreduce
                (
                    MPI_IN_PLACE,
                    array,
                    typeinfo<T>::ncmpt * N,
                    typeinfo<T>::mpicmpttype(),
                    MPI_SUM,
                    groupcomm_
                );
            }
#endif
        }
        
        template <class T> void sum_g (T* array, std::size_t N, int destination) const
        {
#ifdef WITH_MPI
            if (active_ and groupsize_ > 1)
            {
                MPI_Reduce
                (
                    (igroupproc() == destination ? MPI_IN_PLACE : array),
                    (igroupproc() == destination ? array : nullptr),
                    typeinfo<T>::ncmpt * N,
                    typeinfo<T>::mpicmpttype(),
                    MPI_SUM,
                    destination,
                    groupcomm_
                );
            }
#endif
        }
        
        template <class T> void mastersum (T* array, std::size_t N, int destgroup) const
        {
#ifdef WITH_MPI
            if (active_ and IamGroupMaster() and Ngroup_ > 1)
            {
                MPI_Reduce
                (
                    (igroup_ == destgroup ? MPI_IN_PLACE : array),
                    (igroup_ == destgroup ? array : nullptr),
                    typeinfo<T>::ncmpt * N,
                    typeinfo<T>::mpicmpttype(),
                    MPI_SUM,
                    destgroup,
                    mastergroup_
                );
            }
#endif
        }
        
        void wait () const
        {
#ifdef WITH_MPI
            if (active_)
                MPI_Barrier(MPI_COMM_WORLD);
#endif
        }
        
        void wait_g () const
        {
#ifdef WITH_MPI
            if (active_)
                MPI_Barrier(groupcomm_);
#endif
        }
        
        void* groupcomm () const
        {
#ifdef WITH_MPI
            if (active_)
                return (void*)(std::intptr_t)groupcomm_;
#endif
            return MPI_Comm(0);
        }
        
    private:
        
        // whether the MPI is on
        bool active_;
        
        // global communicator rank
        int iproc_;
        
        // global communicator size
        int Nproc_;
        
        // group index
        int igroup_;
        
        // group count
        int Ngroup_;
#ifdef WITH_MPI
        // MPI group communicator
        MPI_Comm groupcomm_;
        
        // MPI master group communicator
        MPI_Comm mastergroup_;
#endif
        // local communicator size
        int groupsize_;
};

#endif