33 #ifndef MADNESS_WORLD_SAFEMPI_H__INCLUDED
34 #define MADNESS_WORLD_SAFEMPI_H__INCLUDED
55 #ifdef MADNESS_MPI_HEADER
56 # include MADNESS_MPI_HEADER
64 #if MADNESS_MPI_THREAD_LEVEL == MPI_THREAD_SERIALIZED
65 # define MADNESS_SERIALIZES_MPI
79 #define MADNESS_MPI_TEST(condition) \
81 int mpi_error_code = condition; \
82 if(mpi_error_code != MPI_SUCCESS) throw ::SafeMPI::Exception(mpi_error_code); \
88 #ifdef MADNESS_SERIALIZES_MPI
89 #define SAFE_MPI_GLOBAL_MUTEX madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
91 #define SAFE_MPI_GLOBAL_MUTEX
136 const int line,
const char* file)
141 std::cerr <<
"!!! MPI ERROR (" << rc <<
") in " <<
function <<
142 " at " << file <<
"(" << line <<
"): " << error_string <<
"\n";
167 std::ostringstream oss;
168 for(
auto s=0; s!=nstatuses; ++s) {
170 auto status_error = statuses[s].MPI_ERROR;
172 oss <<
"request " << indices[s] <<
":";
174 oss <<
" unknown error!" << std::endl;
216 if (
e.can_elaborate()) {
327 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[count]);
330 for(
int i = 0; i < count; ++i)
331 mpi_requests[i] = requests[i].
request_;
337 for(
int i = 0; i < count; ++i)
338 requests[i].
request_ = mpi_requests[i];
345 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[count]);
348 for(
int i = 0; i < count; ++i)
349 mpi_requests[i] = requests[i].
request_;
355 for(
int i = 0; i < count; ++i)
356 requests[i] = mpi_requests[i];
366 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[incount]);
367 std::unique_ptr<MPI_Status[]> mpi_statuses(
new MPI_Status[incount]);
368 for(
int i = 0; i < incount; ++i)
369 mpi_requests[i] = requests[i].
request_;
373 auto mpi_error_code =
375 indices, mpi_statuses.get());
377 throw ::SafeMPI::Exception(mpi_error_code, outcount, indices, mpi_statuses.get());
381 for(
int i = 0; i < incount; ++i) {
382 requests[i] = mpi_requests[i];
383 statuses[i] = mpi_statuses[i];
390 std::unique_ptr<MPI_Request[]> mpi_requests(
new MPI_Request[incount]);
391 for(
int i = 0; i < incount; ++i)
392 mpi_requests[i] = requests[i].
request_;
397 for(
int i = 0; i < incount; ++i)
398 requests[i] = mpi_requests[i];
432 Group result(std::shared_ptr<Impl>(
new Impl(*
pimpl, n,
const_cast<int*
>(ranks))));
440 const_cast<int*
>(ranks1), grp2.
pimpl->group, ranks2));
462 const_cast<int*
>(ranks), &
group));
470 "SafeMPI::Group::Impl::~Impl()", __LINE__, __FILE__);
539 const auto min_tag_value = 1024;
540 const auto max_tag_value = 4094;
541 return max_tag_value - min_tag_value + 1;
554 int result =
urtag++;
555 if (result >= 1000)
MADNESS_EXCEPTION(
"too many reserved tags in use" , result );
576 struct WorldInitObject;
586 int rank = -1, size = -1;
589 take_ownership_of_comm =
591 pimpl.reset(
new Impl(comm, rank, size, take_ownership_of_comm));
616 return Intracomm(std::shared_ptr<Impl>(
new Impl(group_comm,
me, nproc,
true)));
643 return Intracomm(std::shared_ptr<Impl>(
new Impl(group_comm,
me, nproc,
true)));
678 return Intracomm(std::shared_ptr<Impl>(
new Impl(group_comm,
me, nproc,
true)));
721 return pimpl->numproc;
748 void Send(
const void* buf,
const int count,
const MPI_Datatype datatype,
int dest,
int tag)
const {
754 #ifdef MADNESS_USE_BSEND_ACKS
755 void Bsend(
const void* buf,
size_t count,
const MPI_Datatype datatype,
int dest,
int tag)
const {
807 auto rc = sigaction(SIGABRT, NULL, &sa);
808 if (rc == 0 && sa.sa_handler != SIG_DFL) {
832 return pimpl->unique_tag();
847 return pimpl->unique_reserved_tag();
878 inline int Init_thread(
int & argc,
char **& argv,
int requested) {
891 char** argv =
nullptr;
899 inline void Init(
int &argc,
char **&argv) {
907 char** argv =
nullptr;
SafeMPI exception object.
Definition: safempi.h:150
std::string mpi_statuses_error_string_
Definition: safempi.h:153
bool can_elaborate() const noexcept
Definition: safempi.h:207
Exception(const int mpi_error)
Definition: safempi.h:156
char mpi_error_string_[MPI_MAX_ERROR_STRING]
Definition: safempi.h:152
Exception & operator=(const Exception &other)
Definition: safempi.h:196
Exception(const int mpi_error, const int nstatuses, const int *indices, MPI_Status *const statuses) noexcept
Definition: safempi.h:162
friend std::ostream & operator<<(std::ostream &os, const Exception &e)
Definition: safempi.h:214
virtual ~Exception()
Definition: safempi.h:204
virtual const char * what() const
Definition: safempi.h:206
const char * elaborate() const noexcept
Definition: safempi.h:210
Exception(const Exception &other)
Definition: safempi.h:189
Definition: safempi.h:428
Group(const std::shared_ptr< Impl > &p)
Definition: safempi.h:484
Group Incl(int n, const int *ranks) const
Definition: safempi.h:430
MPI_Group group() const
Definition: safempi.h:443
std::shared_ptr< Impl > pimpl
Definition: safempi.h:486
Group(MPI_Comm comm)
Definition: safempi.h:481
void Translate_ranks(int nproc, const int *ranks1, const Group &grp2, int *ranks2) const
Definition: safempi.h:436
Group(const Group &other)
Definition: safempi.h:448
Group()
Definition: safempi.h:478
Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy.
Definition: safempi.h:490
Intracomm(const std::shared_ptr< Impl > &i)
Definition: safempi.h:567
Intracomm Clone() const
Definition: safempi.h:689
void Allreduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const
Definition: safempi.h:787
static bool Comm_compare(const MPI_Comm &comm1, const MPI_Comm &comm2)
Definition: safempi.h:492
bool Get_attr(int key, void *value) const
Definition: safempi.h:792
static const int SHARED_SPLIT_TYPE
Definition: safempi.h:650
Intracomm Create(Group group) const
Definition: safempi.h:609
Request Irecv(void *buf, const int count, const MPI_Datatype datatype, const int src, const int tag) const
Definition: safempi.h:740
void Bcast(void *buf, size_t count, const MPI_Datatype datatype, const int root) const
Definition: safempi.h:775
void Reduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op, const int root) const
Definition: safempi.h:781
Request Isend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition: safempi.h:724
friend int Finalize()
Analogous to MPI_Finalize.
Definition: safempi.h:918
void binary_tree_info(int root, int &parent, int &child0, int &child1)
Construct info about a binary tree with given root.
Definition: safempi.cc:39
Intracomm Split_type(int Type, int Key=0) const
Definition: safempi.h:667
int Get_rank() const
Definition: safempi.h:714
static int unique_tag_period()
Definition: safempi.h:836
void Abort(int code=1) const
Definition: safempi.h:800
static const int UNDEFINED_SPLIT_TYPE
Definition: safempi.h:649
Intracomm(const MPI_Comm &comm, bool take_ownership_of_comm=true)
Definition: safempi.h:582
int unique_tag()
Returns a unique tag for temporary use (1023<tag<4095)
Definition: safempi.h:830
bool operator==(const Intracomm &other) const
Definition: safempi.h:693
void Barrier() const
Definition: safempi.h:815
std::shared_ptr< Impl > pimpl
Definition: safempi.h:560
Group Get_group() const
Definition: safempi.h:702
Intracomm & operator=(const Intracomm &other)
void Send(const void *buf, const int count, const MPI_Datatype datatype, int dest, int tag) const
Definition: safempi.h:748
Intracomm Split(int Color, int Key=0) const
Definition: safempi.h:635
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000)
Definition: safempi.h:845
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag) const
Definition: safempi.h:769
MPI_Comm & Get_mpi_comm() const
Definition: safempi.h:709
~Intracomm()
Definition: safempi.h:596
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag, MPI_Status &status) const
Definition: safempi.h:763
Intracomm(const Intracomm &other)
Definition: safempi.h:594
int Get_size() const
Definition: safempi.h:719
Intracomm()
Definition: safempi.h:573
static const int UNDEFINED_COLOR
Definition: safempi.h:619
Request Issend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition: safempi.h:732
Definition: safempi.h:289
bool operator!=(const Request &other)
Definition: safempi.h:318
Request & operator=(const MPI_Request &other)
Definition: safempi.h:311
MPI_Request request_
Definition: safempi.h:296
bool Test(MPI_Status &status)
Definition: safempi.h:409
bool Test_got_lock_already()
Definition: safempi.h:414
static bool Testany(int count, Request *requests, int &index)
Definition: safempi.h:342
static int Testsome(int incount, Request *requests, int *indices)
Definition: safempi.h:388
Request()
Definition: safempi.h:301
Request(MPI_Request other)
Definition: safempi.h:302
static int Testsome(int incount, Request *requests, int *indices, Status *statuses)
Definition: safempi.h:360
bool Test_got_lock_already(MPI_Status &status)
Definition: safempi.h:403
static bool Testany(int count, Request *requests, int &index, Status &status)
Definition: safempi.h:324
Request & operator=(const Request &other)
Definition: safempi.h:306
bool operator==(const Request &other)
Definition: safempi.h:317
Request(const Request &other)
Definition: safempi.h:303
bool Test()
Definition: safempi.h:420
Definition: safempi.h:224
int Get_count(const MPI_Datatype datatype) const
Definition: safempi.h:262
int Get_error() const
Definition: safempi.h:280
void Set_tag(int tag)
Definition: safempi.h:284
Status & operator=(const Status &other)
Definition: safempi.h:235
Status(const Status &other)
Definition: safempi.h:231
MPI_Status status_
Definition: safempi.h:226
int Get_tag() const
Definition: safempi.h:278
Status(MPI_Status other)
Definition: safempi.h:232
int Get_source() const
Definition: safempi.h:276
Status(void)
Definition: safempi.h:230
void Set_source(int source)
Definition: safempi.h:282
void Set_error(int error)
Definition: safempi.h:286
Status & operator=(const MPI_Status other)
Definition: safempi.h:240
Mutex using pthread mutex operations.
Definition: worldmutex.h:131
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition: derivatives.cc:72
const double m
Definition: gfit.cc:199
Tensor< double > op(const Tensor< double > &x)
Definition: kain.cc:508
Macros and tools pertaining to the configuration of MADNESS.
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition: madness_exception.h:119
#define MADNESS_ASSERT(condition)
Assert a condition that should be free of side-effects since in release builds this might be a no-op.
Definition: madness_exception.h:134
void init_comm_world()
Initialize SafeMPI::COMM_WORLD.
Definition: safempi.h:863
void print_mpi_error(const int rc, const char *function, const int line, const char *file)
Definition: safempi.h:135
Definition: safempi.cc:35
Intracomm COMM_WORLD
Definition: safempi.cc:67
static const int MPIAR_TAG
Definition: safempi.h:105
int Init_thread(int &argc, char **&argv, int requested)
Analogous to MPI_Init_thread.
Definition: safempi.h:878
void Init(int &argc, char **&argv)
Analogous to MPI_Init.
Definition: safempi.h:899
void Op_free(MPI_Op op)
Analogous to MPI_Op_free.
Definition: safempi.h:963
void Attach_buffer(void *buffer, int size)
Set buffer for Bsend .
Definition: safempi.h:942
madness::SCALABLE_MUTEX_TYPE charon
Definition: safempi.cc:37
int Query_thread()
Analogous to MPI_Query_thread.
Definition: safempi.h:927
int Finalize()
Analogous to MPI_Finalize.
Definition: safempi.h:918
double Wtime()
Wall time.
Definition: safempi.h:936
bool Is_initialized()
Check MPI initialization status.
Definition: safempi.h:116
static const int DEFAULT_SEND_RECV_TAG
Definition: safempi.h:106
static const int RMI_TAG
Definition: safempi.h:104
int Detach_buffer(void *&buffer)
Unset the Bsend buffer.
Definition: safempi.h:949
bool Is_finalized()
Check MPI finalization status.
Definition: safempi.h:125
MPI_Op Op_create(MPI_User_function *user_fn, int commute)
Analogous to MPI_Op_create.
Definition: safempi.h:956
bool initialized()
Check if the MADNESS runtime has been initialized (and not subsequently finalized).
Definition: world.cc:74
void error(const char *msg, int code)
Definition: oldtest.cc:57
static const double c
Definition: relops.cc:10
#define MADNESS_MPI_TEST(condition)
Definition: safempi.h:79
#define SAFE_MPI_GLOBAL_MUTEX
Definition: safempi.h:89
Definition: test_dc.cc:47
int MPI_SOURCE
Definition: stubmpi.h:15
int MPI_TAG
Definition: stubmpi.h:16
int MPI_ERROR
Definition: stubmpi.h:17
Definition: safempi.h:452
Impl(MPI_Comm comm)
Definition: safempi.h:455
MPI_Group group
Definition: safempi.h:453
~Impl()
Definition: safempi.h:465
Impl(const Impl &other, int n, const int *ranks)
Definition: safempi.h:459
Definition: safempi.h:498
int numproc
Definition: safempi.h:501
int urtag
Definition: safempi.h:505
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000)
Definition: safempi.h:549
bool owner
Definition: safempi.h:502
static int unique_tag_period()
Definition: safempi.h:538
int me
Definition: safempi.h:500
int utag
Definition: safempi.h:504
MPI_Comm comm
Definition: safempi.h:499
int unique_tag()
Returns a unique tag for temporary use (1023<tag<=4095)
Definition: safempi.h:526
Impl(const MPI_Comm &c, int m, int n, bool o)
Definition: safempi.h:507
~Impl()
Definition: safempi.h:511
#define MPI_SUCCESS
Definition: stubmpi.h:33
int MPI_Op_free(MPI_Op *op)
Definition: stubmpi.h:258
int MPI_Ssend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition: stubmpi.h:167
#define MPI_ERRORS_RETURN
Definition: stubmpi.h:37
void() MPI_User_function(void *a, void *b, int *len, MPI_Datatype *)
Definition: stubmpi.h:107
int MPI_Info_create(MPI_Info *info)
Definition: stubmpi.h:262
int MPI_Bcast(void *, int, MPI_Datatype, int, MPI_Comm)
Definition: stubmpi.h:173
#define MPI_COMM_NULL
Definition: stubmpi.h:47
int MPI_Error_string(int errorcode, char *string, int *resultlen)
Definition: stubmpi.h:228
int MPI_Isend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition: stubmpi.h:164
int MPI_Op
Definition: stubmpi.h:91
int MPI_Group_free(MPI_Group *group)
Definition: stubmpi.h:122
int MPI_Request
Definition: stubmpi.h:11
#define MPI_ERR_IN_STATUS
Definition: stubmpi.h:36
int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm)
Definition: stubmpi.h:187
#define MPI_COMM_WORLD
Definition: stubmpi.h:24
int MPI_Group
Definition: stubmpi.h:10
int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result)
Definition: stubmpi.h:218
int MPI_Finalize()
Definition: stubmpi.h:130
int MPI_Info_free(MPI_Info *info)
Definition: stubmpi.h:263
int MPI_Query_thread(int *provided)
Definition: stubmpi.h:132
int MPI_Comm_create(MPI_Comm, MPI_Group, MPI_Comm *newcomm)
Definition: stubmpi.h:202
#define MPI_REQUEST_NULL
Definition: stubmpi.h:51
int MPI_Op_create(MPI_User_function *user_fn, int commute, MPI_Op *op)
Definition: stubmpi.h:254
int MPI_Finalized(int *flag)
Definition: stubmpi.h:131
#define MPI_STATUS_IGNORE
Definition: stubmpi.h:20
#define MPI_BYTE
Definition: stubmpi.h:74
int MPI_Recv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Status *)
Definition: stubmpi.h:170
#define MPI_STATUSES_IGNORE
Definition: stubmpi.h:21
int MPI_Comm
Definition: stubmpi.h:23
#define MPI_IDENT
Definition: stubmpi.h:41
int MPI_Issend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition: stubmpi.h:165
int MPI_Init_thread(int *, char ***, int, int *provided)
Definition: stubmpi.h:128
unsigned int MPI_Comm_size(MPI_Comm, int *size)
Definition: stubmpi.h:161
int MPI_Buffer_attach(void *, int)
Definition: stubmpi.h:135
int MPI_Comm_set_errhandler(MPI_Comm comm, MPI_Errhandler errhandler)
Definition: stubmpi.h:250
int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key, MPI_Info info, MPI_Comm *newcomm)
Definition: stubmpi.h:193
int MPI_Abort(MPI_Comm, int code)
Definition: stubmpi.h:198
int MPI_Group_translate_ranks(MPI_Group, int, const int[], MPI_Group, int ranks2[])
Definition: stubmpi.h:110
int MPI_Comm_rank(MPI_Comm, int *rank)
Definition: stubmpi.h:160
int MPI_Comm_free(MPI_Comm *comm)
Definition: stubmpi.h:213
double MPI_Wtime()
Definition: stubmpi.h:252
int MPI_Comm_get_attr(MPI_Comm, int, void *, int *)
Definition: stubmpi.h:185
#define MPI_COMM_TYPE_SHARED
Definition: stubmpi.h:60
int MPI_Info
Definition: stubmpi.h:29
int MPI_Allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, MPI_Comm)
Definition: stubmpi.h:180
int MPI_Bsend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition: stubmpi.h:168
int MPI_Irecv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition: stubmpi.h:169
int MPI_Comm_group(MPI_Comm, MPI_Group *group)
Definition: stubmpi.h:208
int MPI_Barrier(MPI_Comm)
Definition: stubmpi.h:200
int MPI_Testsome(int, MPI_Request *, int *outcount, int *, MPI_Status *)
Definition: stubmpi.h:149
int MPI_Get_count(MPI_Status *, MPI_Datatype, int *count)
Definition: stubmpi.h:154
struct MPI_Status MPI_Status
int MPI_Initialized(int *flag)
Definition: stubmpi.h:129
int MPI_Init(int *, char ***)
Definition: stubmpi.h:127
#define MPI_MAX_ERROR_STRING
Definition: stubmpi.h:38
int MPI_Test(MPI_Request *, int *flag, MPI_Status *)
Definition: stubmpi.h:138
int MPI_Group_incl(MPI_Group group, int n, const int ranks[], MPI_Group *newgroup)
Definition: stubmpi.h:117
int MPI_Buffer_detach(void *buffer, int *size)
Definition: stubmpi.h:136
int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, int, MPI_Comm)
Definition: stubmpi.h:176
#define MPI_UNDEFINED
Definition: stubmpi.h:25
int MPI_Datatype
Definition: stubmpi.h:70
int MPI_Testany(int, MPI_Request[], int *index, int *flag, MPI_Status *)
Definition: stubmpi.h:143
int me
Definition: test_binsorter.cc:10
void e()
Definition: test_sig.cc:75
double source(const coordT &r)
Definition: testperiodic.cc:48
const char * status[2]
Definition: testperiodic.cc:43
Implements Mutex, MutexFair, Spinlock, ConditionVariable.