33#ifndef MADNESS_WORLD_SAFEMPI_H__INCLUDED
34#define MADNESS_WORLD_SAFEMPI_H__INCLUDED
55#ifdef MADNESS_MPI_HEADER
56# include MADNESS_MPI_HEADER
64#if MADNESS_MPI_THREAD_LEVEL == MPI_THREAD_SERIALIZED
65# define MADNESS_SERIALIZES_MPI
79#define MADNESS_MPI_TEST(condition) \
81 int mpi_error_code = condition; \
82 if(mpi_error_code != MPI_SUCCESS) throw ::SafeMPI::Exception(mpi_error_code); \
88#ifdef MADNESS_SERIALIZES_MPI
89#define SAFE_MPI_GLOBAL_MUTEX madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
91#define SAFE_MPI_GLOBAL_MUTEX
119 return (initialized != 0);
136 const int line,
const char* file)
141 std::cerr <<
"!!! MPI ERROR (" << rc <<
") in " <<
function <<
142 " at " << file <<
"(" << line <<
"): " << error_string <<
"\n";
167 std::ostringstream oss;
168 for(
auto s=0; s!=nstatuses; ++s) {
170 auto status_error = statuses[s].MPI_ERROR;
172 oss <<
"request " << indices[s] <<
":";
174 oss <<
" unknown error!" << std::endl;
216 if (
e.can_elaborate()) {
327 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[count]);
330 for(
int i = 0; i < count; ++i)
331 mpi_requests[i] = requests[i].
request_;
337 for(
int i = 0; i < count; ++i)
338 requests[i].
request_ = mpi_requests[i];
345 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[count]);
348 for(
int i = 0; i < count; ++i)
349 mpi_requests[i] = requests[i].
request_;
355 for(
int i = 0; i < count; ++i)
356 requests[i] = mpi_requests[i];
366 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[incount]);
367 std::unique_ptr<MPI_Status[]> mpi_statuses(
new MPI_Status[incount]);
368 for(
int i = 0; i < incount; ++i)
369 mpi_requests[i] = requests[i].
request_;
373 auto mpi_error_code =
375 indices, mpi_statuses.get());
377 throw ::SafeMPI::Exception(mpi_error_code, outcount, indices, mpi_statuses.get());
381 for(
int i = 0; i < incount; ++i) {
382 requests[i] = mpi_requests[i];
383 statuses[i] = mpi_statuses[i];
390 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[incount]);
391 for(
int i = 0; i < incount; ++i)
392 mpi_requests[i] = requests[i].
request_;
397 for(
int i = 0; i < incount; ++i)
398 requests[i] = mpi_requests[i];
432 Group result(std::shared_ptr<Impl>(
new Impl(*
pimpl, n,
const_cast<int*
>(ranks))));
440 const_cast<int*
>(ranks1), grp2.
pimpl->group, ranks2));
462 const_cast<int*
>(ranks), &
group));
470 "SafeMPI::Group::Impl::~Impl()", __LINE__, __FILE__);
539 const auto min_tag_value = 1024;
540 const auto max_tag_value = 4094;
541 return max_tag_value - min_tag_value + 1;
554 int result =
urtag++;
555 if (result >= 1000)
MADNESS_EXCEPTION(
"too many reserved tags in use" , result );
576 struct WorldInitObject;
586 int rank = -1, size = -1;
589 take_ownership_of_comm =
591 pimpl.reset(
new Impl(comm, rank, size, take_ownership_of_comm));
616 return Intracomm(std::shared_ptr<Impl>(
new Impl(group_comm,
me, nproc,
true)));
643 return Intracomm(std::shared_ptr<Impl>(
new Impl(group_comm,
me, nproc,
true)));
678 return Intracomm(std::shared_ptr<Impl>(
new Impl(group_comm,
me, nproc,
true)));
721 return pimpl->numproc;
748 void Send(
const void* buf,
const int count,
const MPI_Datatype datatype,
int dest,
int tag)
const {
754#ifdef MADNESS_USE_BSEND_ACKS
755 void Bsend(
const void* buf,
size_t count,
const MPI_Datatype datatype,
int dest,
int tag)
const {
807 auto rc = sigaction(SIGABRT, NULL, &sa);
808 if (rc == 0 && sa.sa_handler != SIG_DFL) {
832 return pimpl->unique_tag();
847 return pimpl->unique_reserved_tag();
878 inline int Init_thread(
int & argc,
char **& argv,
int requested) {
891 char** argv =
nullptr;
899 inline void Init(
int &argc,
char **&argv) {
907 char** argv =
nullptr;
SafeMPI exception object.
Definition safempi.h:150
std::string mpi_statuses_error_string_
Definition safempi.h:153
bool can_elaborate() const noexcept
Definition safempi.h:207
Exception(const int mpi_error)
Definition safempi.h:156
char mpi_error_string_[MPI_MAX_ERROR_STRING]
Definition safempi.h:152
Exception & operator=(const Exception &other)
Definition safempi.h:196
friend std::ostream & operator<<(std::ostream &os, const Exception &e)
Definition safempi.h:214
virtual const char * what() const
Definition safempi.h:206
Exception(const int mpi_error, const int nstatuses, const int *indices, MPI_Status *const statuses) noexcept
Definition safempi.h:162
virtual ~Exception()
Definition safempi.h:204
const char * elaborate() const noexcept
Definition safempi.h:210
Exception(const Exception &other)
Definition safempi.h:189
Group(const std::shared_ptr< Impl > &p)
Definition safempi.h:484
Group Incl(int n, const int *ranks) const
Definition safempi.h:430
MPI_Group group() const
Definition safempi.h:443
std::shared_ptr< Impl > pimpl
Definition safempi.h:486
Group(MPI_Comm comm)
Definition safempi.h:481
void Translate_ranks(int nproc, const int *ranks1, const Group &grp2, int *ranks2) const
Definition safempi.h:436
Group(const Group &other)
Definition safempi.h:448
Group()
Definition safempi.h:478
Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy.
Definition safempi.h:490
Intracomm(const std::shared_ptr< Impl > &i)
Definition safempi.h:567
Intracomm Clone() const
Definition safempi.h:689
void Allreduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const
Definition safempi.h:787
static bool Comm_compare(const MPI_Comm &comm1, const MPI_Comm &comm2)
Definition safempi.h:492
bool Get_attr(int key, void *value) const
Definition safempi.h:792
static const int SHARED_SPLIT_TYPE
Definition safempi.h:650
Intracomm Create(Group group) const
Definition safempi.h:609
Request Irecv(void *buf, const int count, const MPI_Datatype datatype, const int src, const int tag) const
Definition safempi.h:740
void Bcast(void *buf, size_t count, const MPI_Datatype datatype, const int root) const
Definition safempi.h:775
MPI_Comm & Get_mpi_comm() const
Definition safempi.h:709
void Reduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op, const int root) const
Definition safempi.h:781
Request Isend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition safempi.h:724
friend int Finalize()
Analogous to MPI_Finalize.
Definition safempi.h:918
void binary_tree_info(int root, int &parent, int &child0, int &child1)
Construct info about a binary tree with given root.
Definition safempi.cc:39
Intracomm & operator=(const Intracomm &other)
Intracomm Split_type(int Type, int Key=0) const
Definition safempi.h:667
int Get_rank() const
Definition safempi.h:714
static int unique_tag_period()
Definition safempi.h:836
void Abort(int code=1) const
Definition safempi.h:800
static const int UNDEFINED_SPLIT_TYPE
Definition safempi.h:649
Intracomm(const MPI_Comm &comm, bool take_ownership_of_comm=true)
Definition safempi.h:582
int unique_tag()
Returns a unique tag for temporary use (1023<tag<4095)
Definition safempi.h:830
bool operator==(const Intracomm &other) const
Definition safempi.h:693
void Barrier() const
Definition safempi.h:815
std::shared_ptr< Impl > pimpl
Definition safempi.h:560
Group Get_group() const
Definition safempi.h:702
void Send(const void *buf, const int count, const MPI_Datatype datatype, int dest, int tag) const
Definition safempi.h:748
Intracomm Split(int Color, int Key=0) const
Definition safempi.h:635
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000)
Definition safempi.h:845
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag) const
Definition safempi.h:769
~Intracomm()
Definition safempi.h:596
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag, MPI_Status &status) const
Definition safempi.h:763
Intracomm(const Intracomm &other)
Definition safempi.h:594
int Get_size() const
Definition safempi.h:719
Intracomm()
Definition safempi.h:573
static const int UNDEFINED_COLOR
Definition safempi.h:619
Request Issend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition safempi.h:732
bool operator!=(const Request &other)
Definition safempi.h:318
MPI_Request request_
Definition safempi.h:296
bool Test(MPI_Status &status)
Definition safempi.h:409
Request & operator=(const MPI_Request &other)
Definition safempi.h:311
bool Test_got_lock_already()
Definition safempi.h:414
static bool Testany(int count, Request *requests, int &index)
Definition safempi.h:342
static int Testsome(int incount, Request *requests, int *indices)
Definition safempi.h:388
Request()
Definition safempi.h:301
Request(MPI_Request other)
Definition safempi.h:302
static int Testsome(int incount, Request *requests, int *indices, Status *statuses)
Definition safempi.h:360
bool Test_got_lock_already(MPI_Status &status)
Definition safempi.h:403
static bool Testany(int count, Request *requests, int &index, Status &status)
Definition safempi.h:324
Request & operator=(const Request &other)
Definition safempi.h:306
bool operator==(const Request &other)
Definition safempi.h:317
Request(const Request &other)
Definition safempi.h:303
bool Test()
Definition safempi.h:420
int Get_count(const MPI_Datatype datatype) const
Definition safempi.h:262
Status & operator=(const MPI_Status other)
Definition safempi.h:240
int Get_error() const
Definition safempi.h:280
Status & operator=(const Status &other)
Definition safempi.h:235
void Set_tag(int tag)
Definition safempi.h:284
Status(const Status &other)
Definition safempi.h:231
MPI_Status status_
Definition safempi.h:226
int Get_tag() const
Definition safempi.h:278
Status(MPI_Status other)
Definition safempi.h:232
int Get_source() const
Definition safempi.h:276
Status(void)
Definition safempi.h:230
void Set_source(int source)
Definition safempi.h:282
void Set_error(int error)
Definition safempi.h:286
Mutex using pthread mutex operations.
Definition worldmutex.h:131
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition derivatives.cc:72
static double function(const coord_3d &r)
Normalized gaussian.
Definition functionio.cc:100
Tensor< double > op(const Tensor< double > &x)
Definition kain.cc:508
Macros and tools pertaining to the configuration of MADNESS.
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_ASSERT(condition)
Assert a condition that should be free of side-effects since in release builds this might be a no-op.
Definition madness_exception.h:134
void init_comm_world()
Initialize SafeMPI::COMM_WORLD.
Definition safempi.h:863
void print_mpi_error(const int rc, const char *function, const int line, const char *file)
Definition safempi.h:135
Intracomm COMM_WORLD
Definition safempi.cc:67
void Init()
Analogous to MPI_Init.
Definition safempi.h:905
static const int MPIAR_TAG
Definition safempi.h:105
int Init_thread(int &argc, char **&argv, int requested)
Analogous to MPI_Init_thread.
Definition safempi.h:878
void Op_free(MPI_Op op)
Analogous to MPI_Op_free.
Definition safempi.h:963
void Attach_buffer(void *buffer, int size)
Set buffer for Bsend .
Definition safempi.h:942
madness::SCALABLE_MUTEX_TYPE charon
Definition safempi.cc:37
int Query_thread()
Analogous to MPI_Query_thread.
Definition safempi.h:927
int Finalize()
Analogous to MPI_Finalize.
Definition safempi.h:918
double Wtime()
Wall time.
Definition safempi.h:936
bool Is_initialized()
Check MPI initialization status.
Definition safempi.h:116
static const int DEFAULT_SEND_RECV_TAG
Definition safempi.h:106
static const int RMI_TAG
Definition safempi.h:104
int Detach_buffer(void *&buffer)
Unset the Bsend buffer.
Definition safempi.h:949
bool Is_finalized()
Check MPI finalization status.
Definition safempi.h:125
MPI_Op Op_create(MPI_User_function *user_fn, int commute)
Analogous to MPI_Op_create.
Definition safempi.h:956
void error(const char *msg, int code)
Definition oldtest.cc:57
static const double c
Definition relops.cc:10
static const double m
Definition relops.cc:9
#define MADNESS_MPI_TEST(condition)
Definition safempi.h:79
#define SAFE_MPI_GLOBAL_MUTEX
Definition safempi.h:89
int MPI_SOURCE
Definition stubmpi.h:16
int MPI_TAG
Definition stubmpi.h:17
int MPI_ERROR
Definition stubmpi.h:18
Impl(MPI_Comm comm)
Definition safempi.h:455
MPI_Group group
Definition safempi.h:453
~Impl()
Definition safempi.h:465
Impl(const Impl &other, int n, const int *ranks)
Definition safempi.h:459
int numproc
Definition safempi.h:501
int urtag
Definition safempi.h:505
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000)
Definition safempi.h:549
bool owner
Definition safempi.h:502
static int unique_tag_period()
Definition safempi.h:538
int me
Definition safempi.h:500
int utag
Definition safempi.h:504
MPI_Comm comm
Definition safempi.h:499
int unique_tag()
Returns a unique tag for temporary use (1023<tag<=4095)
Definition safempi.h:526
Impl(const MPI_Comm &c, int m, int n, bool o)
Definition safempi.h:507
~Impl()
Definition safempi.h:511
#define MPI_SUCCESS
Definition stubmpi.h:36
int MPI_Op_free(MPI_Op *op)
Definition stubmpi.h:326
int MPI_Ssend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition stubmpi.h:212
#define MPI_ERRORS_RETURN
Definition stubmpi.h:40
void() MPI_User_function(void *a, void *b, int *len, MPI_Datatype *)
Definition stubmpi.h:152
int MPI_Info_create(MPI_Info *info)
Definition stubmpi.h:330
int MPI_Bcast(void *, int, MPI_Datatype, int, MPI_Comm)
Definition stubmpi.h:241
#define MPI_COMM_NULL
Definition stubmpi.h:50
int MPI_Error_string(int errorcode, char *string, int *resultlen)
Definition stubmpi.h:296
int MPI_Isend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition stubmpi.h:209
int MPI_Op
Definition stubmpi.h:136
int MPI_Group_free(MPI_Group *group)
Definition stubmpi.h:167
int MPI_Request
Definition stubmpi.h:12
#define MPI_ERR_IN_STATUS
Definition stubmpi.h:39
int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm)
Definition stubmpi.h:255
#define MPI_COMM_WORLD
Definition stubmpi.h:25
int MPI_Group
Definition stubmpi.h:11
int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result)
Definition stubmpi.h:286
int MPI_Finalize()
Definition stubmpi.h:175
int MPI_Info_free(MPI_Info *info)
Definition stubmpi.h:331
int MPI_Query_thread(int *provided)
Definition stubmpi.h:177
int MPI_Comm_create(MPI_Comm, MPI_Group, MPI_Comm *newcomm)
Definition stubmpi.h:270
#define MPI_REQUEST_NULL
Definition stubmpi.h:54
int MPI_Op_create(MPI_User_function *user_fn, int commute, MPI_Op *op)
Definition stubmpi.h:322
int MPI_Finalized(int *flag)
Definition stubmpi.h:176
#define MPI_STATUS_IGNORE
Definition stubmpi.h:21
#define MPI_BYTE
Definition stubmpi.h:77
int MPI_Recv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Status *)
Definition stubmpi.h:215
#define MPI_STATUSES_IGNORE
Definition stubmpi.h:22
int MPI_Comm
Definition stubmpi.h:24
#define MPI_IDENT
Definition stubmpi.h:44
int MPI_Issend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition stubmpi.h:210
int MPI_Init_thread(int *, char ***, int, int *provided)
Definition stubmpi.h:173
unsigned int MPI_Comm_size(MPI_Comm, int *size)
Definition stubmpi.h:206
int MPI_Buffer_attach(void *, int)
Definition stubmpi.h:180
int MPI_Comm_set_errhandler(MPI_Comm comm, MPI_Errhandler errhandler)
Definition stubmpi.h:318
int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key, MPI_Info info, MPI_Comm *newcomm)
Definition stubmpi.h:261
int MPI_Abort(MPI_Comm, int code)
Definition stubmpi.h:266
int MPI_Group_translate_ranks(MPI_Group, int, const int[], MPI_Group, int ranks2[])
Definition stubmpi.h:155
int MPI_Comm_rank(MPI_Comm, int *rank)
Definition stubmpi.h:205
int MPI_Comm_free(MPI_Comm *comm)
Definition stubmpi.h:281
double MPI_Wtime()
Definition stubmpi.h:320
int MPI_Comm_get_attr(MPI_Comm, int, void *, int *)
Definition stubmpi.h:253
#define MPI_COMM_TYPE_SHARED
Definition stubmpi.h:63
int MPI_Info
Definition stubmpi.h:30
int MPI_Allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, MPI_Comm)
Definition stubmpi.h:248
int MPI_Bsend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition stubmpi.h:213
int MPI_Irecv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition stubmpi.h:214
int MPI_Comm_group(MPI_Comm, MPI_Group *group)
Definition stubmpi.h:276
int MPI_Barrier(MPI_Comm)
Definition stubmpi.h:268
int MPI_Testsome(int, MPI_Request *, int *outcount, int *, MPI_Status *)
Definition stubmpi.h:194
int MPI_Get_count(MPI_Status *, MPI_Datatype, int *count)
Definition stubmpi.h:199
int MPI_Initialized(int *flag)
Definition stubmpi.h:174
int MPI_Init(int *, char ***)
Definition stubmpi.h:172
#define MPI_MAX_ERROR_STRING
Definition stubmpi.h:41
int MPI_Test(MPI_Request *, int *flag, MPI_Status *)
Definition stubmpi.h:183
int MPI_Group_incl(MPI_Group group, int n, const int ranks[], MPI_Group *newgroup)
Definition stubmpi.h:162
int MPI_Buffer_detach(void *buffer, int *size)
Definition stubmpi.h:181
int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, int, MPI_Comm)
Definition stubmpi.h:244
#define MPI_UNDEFINED
Definition stubmpi.h:26
int MPI_Datatype
Definition stubmpi.h:73
int MPI_Testany(int, MPI_Request[], int *index, int *flag, MPI_Status *)
Definition stubmpi.h:188
int me
Definition test_binsorter.cc:10
void e()
Definition test_sig.cc:75
double source(const coordT &r)
Definition testperiodic.cc:48
const char * status[2]
Definition testperiodic.cc:43
Implements Mutex, MutexFair, Spinlock, ConditionVariable.