MpiContainer.h

Go to the documentation of this file.

// This file is part of the MercuryDPM project (https://www.mercurydpm.org).
// Copyright (c), The MercuryDPM Developers Team. All rights reserved.
// License: BSD 3-Clause License; see the LICENSE file in the root directory.
 
#ifndef MPICONTAINER_H_
#define MPICONTAINER_H_
 
#include <cstddef>
#include <functional>
#include <vector>
#include "Math/Vector.h"
 
#ifdef MERCURYDPM_USE_MPI
#include <mpi.h>
#endif
 
#ifdef MERCURYDPM_FORCE_ASSERTS
#define MERCURYDPM_ASSERTS true
#else
#ifdef MERCURYDPM_NO_ASSERTS
#define MERCURYDPM_ASSERTS false
#else
#ifdef NDEBUG
#define MERCURYDPM_ASSERTS false
#else
#define MERCURYDPM_ASSERTS true
#endif
#endif
#endif
 
class SpeciesHandler;
 
enum MercuryMPIType
{
    PARTICLE = 0, POSITION = 1, VELOCITY = 2, FORCE = 3, INTERACTION = 4, SUPERQUADRIC = 5
};
 
enum MercuryMPITag
{
    PARTICLE_COUNT = 0,
    PARTICLE_DATA = 1,
    POSITION_DATA = 2,
    PERIODIC_POSITION_DATA = 3,
    VELOCITY_DATA = 4,
    INTERACTION_COUNT = 5,
    INTERACTION_DATA = 6,
    PERIODIC_COMPLEXITY = 7,
    PARTICLE_INDEX = 8,
    SUPERQUADRIC_DATA = 9
};
 
namespace Detail
{
#ifdef MERCURYDPM_USE_MPI
//convert integral data to the corresponding MPI type
template<typename T>
typename std::enable_if<std::is_integral<T>::value, MPI_Datatype>::type
toMPIType(T t)
{
    MPI_Datatype type;
    MPI_Type_match_size(MPI_TYPECLASS_INTEGER, sizeof(T), &type);
    return type;
}
 
//convert floating point data to the corresponding MPI type
template<typename T>
typename std::enable_if<std::is_floating_point<T>::value, MPI_Datatype>::type
toMPIType(T t)
{
    MPI_Datatype type;
    MPI_Type_match_size(MPI_TYPECLASS_REAL,sizeof(T),&type);
    return type;
}
#endif
} /*detail*/
 
void initialiseMPI();
 
class MPIContainer final
{
public:
    static MPIContainer& Instance()
    {
        static MPIContainer theInstance;
        return theInstance;
    }
    
    void initialiseMercuryMPITypes(const SpeciesHandler& speciesHandler);
    
    void sync()
    {
#ifdef MERCURYDPM_USE_MPI
        MPI_Waitall(pending_.size(),pending_.data(),MPI_STATUSES_IGNORE);
        pending_.clear();
        MPI_Barrier(communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    send(T& t, int to, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Request request;
        MPI_Isend(&t, 1, Detail::toMPIType(t), to, tag, communicator_, &request);
        pending_.push_back(request);
 
#endif
    }
 
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    send(T* t, int count, int to, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Request request;
        MPI_Isend(t, count, Detail::toMPIType(*t), to, tag, communicator_, &request);
        pending_.push_back(request);
 
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    receive(T& t, int from, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data from self!");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Request request;
        MPI_Irecv(&t, 1, Detail::toMPIType(t), from, tag, communicator_, &request);
        pending_.push_back(request);
 
#endif
    }
 
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    receive(T* t, int count, int from, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data fromself!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Request request;
        MPI_Irecv(&t, count, Detail::toMPIType(*t), from, tag, communicator_, &request);
        pending_.push_back(request);
 
#endif
    }
    
    
    template<typename T>
    void send(T* t, MercuryMPIType type, int count, int to, int tag)
    {
        //std::cout << "[Process: " << processorID_ << "]: QUEUING SEND REQUEST with tag: " << tag << std::endl;
#if MERCURYDPM_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Request request;
        MPI_Isend(t, count, dataTypes_[type], to, tag, communicator_, &request);
        pending_.push_back(request);
 
 
#endif
    }
    
    template<typename T>
    void receive(T* t, MercuryMPIType type, int count, int from, int tag)
    {
        //std::cout << "[Process: " << processorID_ << "]: QUEUING RECEIVE REQUEST with tag: " << tag << std::endl;
#if MERCURYDPM_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Request request;
        MPI_Irecv(t, count, dataTypes_[type], from, tag, communicator_, &request);
        pending_.push_back(request);
 
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    directSend(T& t, int count, int to, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Ssend(&t, count, Detail::toMPIType(t), to, tag, communicator_);
#endif
    }
    
    
    template<typename T>
    void directSend(T* t, MercuryMPIType type, int count, int to, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Ssend(t,count,dataTypes_[type], to, tag, communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    directReceive(T& t, int count, int from, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data from self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Recv(&t, count, Detail::toMPIType(t), from, tag,communicator_, MPI_STATUS_IGNORE);
#endif
    }
    
    template<typename T>
    void directReceive(T* t, MercuryMPIType type, int count, int from, int tag)
    {
#if MERCURYDPM_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURYDPM_USE_MPI
        MPI_Recv(t, count, dataTypes_[type], from, tag, communicator_, MPI_STATUS_IGNORE);
#endif
    }
    
    template<typename T>
    void gather(T& send_t, T* receive_t)
    {
#ifdef MERCURYDPM_USE_MPI
        MPI_Gather(&send_t, 1, Detail::toMPIType(send_t), receive_t, 1, Detail::toMPIType(send_t), 0, communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    broadcast(T& t, int fromProcessor = 0)
    {
#ifdef MERCURYDPM_USE_MPI
        MPI_Bcast(&t,1,Detail::toMPIType(t),fromProcessor,communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    broadcast(T* t, int size, int fromProcessor)
    {
#ifdef MERCURYDPM_USE_MPI
        MPI_Bcast((void *)t,size,Detail::toMPIType(t[0]),fromProcessor,communicator_);
 
#endif
    }
    
    template<typename T>
    void broadcast(T* t, MercuryMPIType type, int fromProcessor = 0)
    {
#ifdef MERCURYDPM_USE_MPI
        MPI_Bcast((void *)t,1,dataTypes_[type],fromProcessor,communicator_);
#endif
    }
    
#ifdef MERCURYDPM_USE_MPI
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    reduce(T& t, MPI_Op operation, int id = 0)
    {
 
        if(id == getProcessorID())
        {
            MPI_Reduce(MPI_IN_PLACE, &t, 1, Detail::toMPIType(t), operation, id, communicator_);
        }
        else
        {
            MPI_Reduce(&t, nullptr, 1, Detail::toMPIType(t), operation, id, communicator_);
        }
    }
#endif
    
#ifdef MERCURYDPM_USE_MPI
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    allReduce(T& send_t, T& receive_t, MPI_Op operation)
    {
        MPI_Allreduce(&send_t, &receive_t, 1, Detail::toMPIType(send_t), operation,communicator_);  
    }
#endif
    
#ifdef MERCURYDPM_USE_MPI
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    allGather(T& send_t, int send_count, std::vector<T>& receive_t, int receive_count)
    {
        MPI_Allgather(&send_t, send_count, Detail::toMPIType(send_t),
                                receive_t.data(), receive_count, Detail::toMPIType(receive_t[0]),communicator_);  
    }
#endif
    
    std::size_t getProcessorID();
    
    std::size_t getNumberOfProcessors() const;
    
#ifdef MERCURYDPM_USE_MPI
    MPI_Comm& getComm();
#endif
    
    template<typename T>
    void createMercuryMPIType(T t, MercuryMPIType type)
    {
#ifdef MERCURYDPM_USE_MPI
        MPI_Datatype MPIType;
        MPI_Type_contiguous(sizeof(T), MPI_BYTE, &MPIType);
        MPI_Type_commit(&MPIType);
        dataTypes_.push_back(MPIType);
#endif
    }
    
    
    void deleteMercuryMPITypes()
    {
#ifdef MERCURYDPM_USE_MPI
        for(MPI_Datatype type : dataTypes_)
        {
            MPI_Type_free(&type);
        }
#endif
    }
    
    MPIContainer(const MPIContainer& orig) = delete;
 
private:
    
    MPIContainer();
    
    //std::size_t processorID_;
    int processorID_;
    
    int numberOfProcessors_;
 
#ifdef MERCURYDPM_USE_MPI
    std::vector<MPI_Request> pending_;
    
    MPI_Comm communicator_;
    std::vector<MPI_Datatype> dataTypes_;
 
#endif
 
};
 
 
#endif /* MPICONTAINER_H_ */