intercepts.cc

MPI stands for the Message Passing Interface. Written by the MPI Forum (a large committee comprising

// function because its signature includes a (MPI::Datatype&), which
// we can't put in the C library libmpi.
//
// 5. When the user invokes an function that uses the MPI::Op (or,
// more specifically, when the Op dispatch engine in ompi/op/op.c [in
// libmpi] tries to dispatch off to it), it will see the
// OMPI_OP_FLAGS_CXX_FUNC flag and know to use the
// op->o_func[0].cxx_intercept_fn and also pass as the 4th argument,
// op->o_func[1].c_fn.
//
// 6. ompi_mpi_cxx_op_intercept() is therefore invoked and receives
// both the (MPI_Datatype*) (which is easy to convert to
// (MPI::Datatype&)) and a pointer to the user's C++ callback function
// (albiet cast as the wrong type).  So it casts the callback function
// pointer to (MPI::User_function*) and invokes it.
//
// Wasn't that simple?
//
extern "C" void
ompi_mpi_cxx_op_intercept(void *invec, void *outvec, int *len, 
                          MPI_Datatype *datatype, MPI_User_function *c_fn)
{
    MPI::Datatype cxx_datatype = *datatype;
    MPI::User_function *cxx_callback = (MPI::User_function*) c_fn;
    cxx_callback(invec, outvec, *len, cxx_datatype);
}

//
// Attribute copy functions -- comm, type, and win
//
extern "C" int
ompi_mpi_cxx_comm_copy_attr_intercept(MPI_Comm comm, int keyval, 
                                      void *extra_state, void *attribute_val_in, 
                                      void *attribute_val_out, int *flag,
                                      MPI_Comm newcomm)
{
  int ret = 0;
  MPI::Comm::keyval_pair_t* copy_and_delete; 
  MPI::Comm::Copy_attr_function* copy_fn;

  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
  copy_and_delete = MPI::Comm::mpi_comm_keyval_fn_map[keyval];
  copy_fn = copy_and_delete->first;
  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);
  
  MPI::Intracomm intracomm;
  MPI::Intercomm intercomm;
  MPI::Graphcomm graphcomm;
  MPI::Cartcomm cartcomm;
  
  bool bflag = OPAL_INT_TO_BOOL(*flag); 

  if (NULL != copy_fn) {
      if (OMPI_COMM_IS_GRAPH(comm)) {
          graphcomm = MPI::Graphcomm(comm);
          ret = copy_fn(graphcomm, keyval, extra_state,
                        attribute_val_in, attribute_val_out, bflag);
      } else if (OMPI_COMM_IS_CART(comm)) {
          cartcomm = MPI::Cartcomm(comm);
          ret = copy_fn(cartcomm, keyval, extra_state,
                        attribute_val_in, attribute_val_out, bflag);
      } else if (OMPI_COMM_IS_INTRA(comm)) {
          intracomm = MPI::Intracomm(comm);
          ret = copy_fn(intracomm, keyval, extra_state,
                        attribute_val_in, attribute_val_out, bflag);
      } else if (OMPI_COMM_IS_INTER(comm)) {
          intercomm = MPI::Intercomm(comm);
          ret = copy_fn(intercomm, keyval, extra_state,
                        attribute_val_in, attribute_val_out, bflag);
      } else {
          ret = MPI::ERR_COMM;
      }
  } else {
      ret = MPI::ERR_OTHER;
  }

  *flag = (int)bflag;
  return ret;
}

extern "C" int
ompi_mpi_cxx_comm_delete_attr_intercept(MPI_Comm comm, int keyval, 
                                        void *attribute_val, void *extra_state)
{
  int ret = 0;
  MPI::Comm::keyval_pair_t * copy_and_delete;
  MPI::Comm::Delete_attr_function* delete_fn;  

  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
  copy_and_delete = MPI::Comm::mpi_comm_keyval_fn_map[keyval];
  delete_fn = copy_and_delete->second;
  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);

  MPI::Intracomm intracomm;
  MPI::Intercomm intercomm;
  MPI::Graphcomm graphcomm;
  MPI::Cartcomm cartcomm;
  
  if (NULL != delete_fn) {
      if (OMPI_COMM_IS_GRAPH(comm)) {
          graphcomm = MPI::Graphcomm(comm);
          ret = delete_fn(graphcomm, keyval, attribute_val, extra_state);
      } else if (OMPI_COMM_IS_CART(comm)) {
          cartcomm = MPI::Cartcomm(comm);
          ret = delete_fn(cartcomm, keyval, attribute_val, extra_state);
      } else if (OMPI_COMM_IS_INTRA(comm)) {
          intracomm = MPI::Intracomm(comm);
          ret = delete_fn(intracomm, keyval, attribute_val, extra_state);
      } else if (OMPI_COMM_IS_INTER(comm)) {
          intercomm = MPI::Intercomm(comm);
          ret = delete_fn(intercomm, keyval, attribute_val, extra_state);
      } else {
          ret = MPI::ERR_COMM;
      }
  } else {
      ret = MPI::ERR_OTHER;
  }

  return ret; 
}


extern "C" int
ompi_mpi_cxx_type_copy_attr_intercept(MPI_Datatype oldtype, int keyval, 
                                      void *extra_state, void *attribute_val_in, 
                                      void *attribute_val_out, int *flag)
{
  int ret = 0;

  MPI::Datatype::keyval_pair_t* copy_and_delete;
  MPI::Datatype::Copy_attr_function* copy_fn;
  MPI::Datatype *cxx_oldtype;

  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
  cxx_oldtype = MPI::Datatype::mpi_type_map[oldtype];
  copy_and_delete = MPI::Datatype::mpi_type_keyval_fn_map[keyval];
  copy_fn = copy_and_delete->first;
  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);

  bool bflag = OPAL_INT_TO_BOOL(*flag); 

  ret = copy_fn(*cxx_oldtype, keyval, extra_state, attribute_val_in, 
                attribute_val_out, bflag);

  *flag = (int)bflag;
  return ret;
}

extern "C" int
ompi_mpi_cxx_type_delete_attr_intercept(MPI_Datatype type, int keyval, 
                                        void *attribute_val, void *extra_state)
{
  int ret = 0;

  MPI::Datatype::keyval_pair_t* copy_and_delete;
  MPI::Datatype::Delete_attr_function* delete_fn;
  MPI::Datatype *cxx_type;

  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
  cxx_type = MPI::Datatype::mpi_type_map[type];
  copy_and_delete = MPI::Datatype::mpi_type_keyval_fn_map[keyval];
  delete_fn = copy_and_delete->second;
  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);

  ret = delete_fn(*cxx_type, keyval, attribute_val, extra_state); 
  return ret;
}

extern "C" int
ompi_mpi_cxx_win_copy_attr_intercept(MPI_Win oldwin, int keyval, 
                                      void *extra_state, void *attribute_val_in, 
                                      void *attribute_val_out, int *flag)
{
  int ret = 0;

  MPI::Win::keyval_pair_t* copy_and_delete;
  MPI::Win::Copy_attr_function* copy_fn;
  MPI::Win *cxx_oldwin;

  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
  cxx_oldwin = MPI::Win::mpi_win_map[oldwin];
  copy_and_delete = MPI::Win::mpi_win_keyval_fn_map[keyval];
  copy_fn = copy_and_delete->first;
  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);

  bool bflag = OPAL_INT_TO_BOOL(*flag); 

  ret = copy_fn(*cxx_oldwin, keyval, extra_state, attribute_val_in, 
                attribute_val_out, bflag);

  *flag = (int)bflag;
  return ret;
}

extern "C" int
ompi_mpi_cxx_win_delete_attr_intercept(MPI_Win win, int keyval, 
                                        void *attribute_val, void *extra_state)
{
  int ret = 0;

  MPI::Win::keyval_pair_t* copy_and_delete;
  MPI::Win::Delete_attr_function* delete_fn;
  MPI::Win *cxx_win;

  OPAL_THREAD_LOCK(MPI::mpi_map_mutex);
  cxx_win = MPI::Win::mpi_win_map[win];
  copy_and_delete = MPI::Win::mpi_win_keyval_fn_map[keyval];
  delete_fn = copy_and_delete->second;
  OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex);

  ret = delete_fn(*cxx_win, keyval, attribute_val, extra_state); 
  return ret;
}


// For similar reasons as above, we need to intercept calls for the 3
// generalized request callbacks (convert arguments to C++ types and
// invoke the C++ callback signature).

extern "C" int
ompi_mpi_cxx_grequest_query_fn_intercept(void *state, MPI_Status *status)
{
    struct MPI::Grequest_intercept_t *data = 
        (struct MPI::Grequest_intercept_t *) state;

    MPI::Status s(*status);
    int ret = data->git_cxx_query_fn(data->git_extra, s);
    *status = s;
    return ret;
}

extern "C" int
ompi_mpi_cxx_grequest_free_fn_intercept(void *state)
{
    struct MPI::Grequest_intercept_t *data = 
        (struct MPI::Grequest_intercept_t *) state;
    int ret = data->git_cxx_free_fn(data->git_extra);
    // Delete the struct that was "new"ed in MPI::Grequest::Start()
    delete data;
    return ret;
}

extern "C" int
ompi_mpi_cxx_grequest_cancel_fn_intercept(void *state, int cancelled)
{
    struct MPI::Grequest_intercept_t *data = 
        (struct MPI::Grequest_intercept_t *) state;
    return data->git_cxx_cancel_fn(data->git_extra, (bool) cancelled);
}