mpimanager.cpp

open lattice boltzmann project www.openlb.org

/*  This file is part of the OpenLB library
 *
 *  Copyright (C) 2007 The OpenLB project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public 
 *  License along with this program; if not, write to the Free 
 *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA  02110-1301, USA.
*/

/** \file
 * Wrapper functions that simplify the use of MPI, template instatiations
 */

#ifdef PARALLEL_MODE_MPI

#include "mpiManager.h"
#include <iostream>

namespace olb {

namespace singleton {

MpiManager::MpiManager() : ok(false)
{ }

MpiManager::~MpiManager() {
    if (ok) {
        MPI_Finalize();
        ok = false;
    }
}

void MpiManager::init(int *argc, char ***argv, bool verbous) {
    if (verbous) {
        std::cerr << "Constructing an MPI thread" << std::endl;
    }
    int ok1 = MPI_Init(argc, argv);
    int ok2 = MPI_Comm_rank(MPI_COMM_WORLD,&taskId);
    int ok3 = MPI_Comm_size(MPI_COMM_WORLD,&numTasks);
    ok = (ok1==0 && ok2==0 && ok3==0);
}

int MpiManager::getSize() const {
    return numTasks;
}

int MpiManager::getRank() const {
    return taskId;
}

int MpiManager::bossId() const {
    return 0;
}

bool MpiManager::isMainProcessor() const {
    return bossId() == getRank();
}

double MpiManager::getTime() const {
    if (!ok) return 0.;
    return MPI_Wtime();
}

void MpiManager::barrier(MPI_Comm comm) {
    if (!ok) return;
    MPI_Barrier(comm);
}

template <>
void MpiManager::send<char>(char *buf, int count, int dest, int tag, MPI_Comm comm) {
    if (!ok) return;
    MPI_Send(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm);
}

template <>
void MpiManager::send<int>(int *buf, int count, int dest, int tag, MPI_Comm comm) {
    if (!ok) return;
    MPI_Send(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm);
}

template <>
void MpiManager::send<float>(float *buf, int count, int dest, int tag, MPI_Comm comm) {
    if (!ok) return;
    MPI_Send(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm);
}

template <>
void MpiManager::send<double>(double *buf, int count, int dest, int tag, MPI_Comm comm) {
    if (!ok) return;
    MPI_Send(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm);
}

template <>
void MpiManager::iSend<char>
    (char *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm, request);
    }
}

template <>
void MpiManager::iSend<int>
    (int *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm, request);
    }
}

template <>
void MpiManager::iSend<float>
    (float *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm, request);
    }
}

template <>
void MpiManager::iSend<double>
    (double *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm, request);
    }
}

template <>
void MpiManager::iSendRequestFree<char>
    (char *buf, int count, int dest, int tag, MPI_Comm comm)
{
    MPI_Request request;
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm, &request);
    }
    MPI_Request_free(&request);
}

template <>
void MpiManager::iSendRequestFree<int>
    (int *buf, int count, int dest, int tag, MPI_Comm comm)
{
    MPI_Request request;
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm, &request);
    }
    MPI_Request_free(&request);
}

template <>
void MpiManager::iSendRequestFree<float>
    (float *buf, int count, int dest, int tag, MPI_Comm comm)
{
    MPI_Request request;
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm, &request);
    }
    MPI_Request_free(&request);
}

template <>
void MpiManager::iSendRequestFree<double>
    (double *buf, int count, int dest, int tag, MPI_Comm comm)
{
    MPI_Request request;
    if (ok) {
        MPI_Isend(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm, &request);
    }
    MPI_Request_free(&request);
}

template <>
void MpiManager::receive<char>(char *buf, int count, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Recv(static_cast<void*>(buf), count, MPI_CHAR, source, tag, comm, &status);
}

template <>
void MpiManager::receive<int>(int *buf, int count, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Recv(static_cast<void*>(buf), count, MPI_INT, source, tag, comm, &status);
}

template <>
void MpiManager::receive<float>(float *buf, int count, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Recv(static_cast<void*>(buf), count, MPI_FLOAT, source, tag, comm, &status);
}

template <>
void MpiManager::receive<double>(double *buf, int count, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Recv(static_cast<void*>(buf), count, MPI_DOUBLE, source, tag, comm, &status);
}

template <>
void MpiManager::sendToMaster<char>(char* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
{
    if (!ok) return;
    if (iAmRoot && !isMainProcessor()) {
        send(sendBuf, sendCount, 0);
    }
    if (isMainProcessor() && !iAmRoot) {
        receive(sendBuf, sendCount, MPI_ANY_SOURCE);
    }
}

template <>
void MpiManager::sendToMaster<int>(int* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
{
    if (!ok) return;
    if (iAmRoot && !isMainProcessor()) {
        send(sendBuf, sendCount, 0);
    }
    if (isMainProcessor() && !iAmRoot) {
        receive(sendBuf, sendCount, MPI_ANY_SOURCE);
    }
}

template <>
void MpiManager::sendToMaster<float>(float* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
{
    if (!ok) return;
    if (iAmRoot && !isMainProcessor()) {
        send(sendBuf, sendCount, 0);
    }
    if (isMainProcessor() && !iAmRoot) {
        receive(sendBuf, sendCount, MPI_ANY_SOURCE);
    }
}

template <>
void MpiManager::sendToMaster<double>(double* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
{
    if (!ok) return;
    if (iAmRoot && !isMainProcessor()) {
        send(sendBuf, sendCount, 0);
    }
    if (isMainProcessor() && !iAmRoot) {
        receive(sendBuf, sendCount, MPI_ANY_SOURCE);
    }
}


template <>
void MpiManager::iRecv<char>(char *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
      MPI_Irecv(static_cast<void*>(buf), count, MPI_CHAR, source, tag, comm, request);
    }
}

template <>
void MpiManager::iRecv<int>(int *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
      MPI_Irecv(static_cast<void*>(buf), count, MPI_INT, source, tag, comm, request);
    }
}

template <>
void MpiManager::iRecv<float>(float *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
      MPI_Irecv(static_cast<void*>(buf), count, MPI_FLOAT, source, tag, comm, request);
    }
}

template <>
void MpiManager::iRecv<double>(double *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
{
    if (ok) {
      MPI_Irecv(static_cast<void*>(buf), count, MPI_DOUBLE, source, tag, comm, request);
    }
}

template <>
void MpiManager::sendRecv<char>
    (char *sendBuf, char *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Sendrecv(static_cast<void*>(sendBuf),
                 count,
                 MPI_CHAR, dest, tag,
                 static_cast<void*>(recvBuf),
                 count,
                 MPI_CHAR, source, tag, comm, &status);
}

template <>
void MpiManager::sendRecv<int>
    (int *sendBuf, int *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Sendrecv(static_cast<void*>(sendBuf),
                 count,
                 MPI_INT, dest, tag,
                 static_cast<void*>(recvBuf),
                 count,
                 MPI_INT, source, tag, comm, &status);
}

template <>
void MpiManager::sendRecv<float>
    (float *sendBuf, float *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Sendrecv(static_cast<void*>(sendBuf),
                 count,
                 MPI_FLOAT, dest, tag,
                 static_cast<void*>(recvBuf),
                 count,
                 MPI_FLOAT, source, tag, comm, &status);
}

template <>
void MpiManager::sendRecv<double>
    (double *sendBuf, double *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Status status;
    MPI_Sendrecv(static_cast<void*>(sendBuf),
                 count,
                 MPI_DOUBLE, dest, tag,
                 static_cast<void*>(recvBuf),
                 count,
                 MPI_DOUBLE, source, tag, comm, &status);
}

template <>
void MpiManager::scatterv_impl<char>(char* sendBuf, int* sendCounts, int* displs,
                                     char* recvBuf, int recvCount, int root, MPI_Comm comm)
{
    if (!ok) return;
    MPI_Scatterv(static_cast<void*>(sendBuf),
                 sendCounts, displs, MPI_CHAR,
                 static_cast<void*>(recvBuf),
                 recvCount, MPI_CHAR, root, comm);