MADNESS 0.10.1
safempi.h
Go to the documentation of this file.
1/*
2 This file is part of MADNESS.
3
4 Copyright (C) 2007,2010 Oak Ridge National Laboratory
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
20 For more information please contact:
21
22 Robert J. Harrison
23 Oak Ridge National Laboratory
24 One Bethel Valley Road
25 P.O. Box 2008, MS-6367
26
27 email: harrisonrj@ornl.gov
28 tel: 865-241-3937
29 fax: 865-572-0680
30
31 $Id$
32*/
33#ifndef MADNESS_WORLD_SAFEMPI_H__INCLUDED
34#define MADNESS_WORLD_SAFEMPI_H__INCLUDED
35
36/// \file safempi.h
37/// \brief Serializes calls to MPI in case it does not support THREAD_MULTIPLE
38
40
41#ifdef STUBOUTMPI
43#else
44
45//#ifdef SEEK_SET
46//#undef SEEK_SET
47//#endif
48//#ifdef SEEK_CUR
49//#undef SEEK_CUR
50//#endif
51//#ifdef SEEK_END
52//#undef SEEK_END
53//#endif
54
55#ifdef MADNESS_MPI_HEADER
56# include MADNESS_MPI_HEADER
57#else
58# include <mpi.h>
59#endif
60
61#endif
62
63/// Defined if MPI has the large-count functions introduced by MPI 4 ( \c MPI_Isend_c
64/// and the like), which take counts of type \c MPI_Count rather than \c int
65#if !defined(STUBOUTMPI) && defined(MPI_VERSION) && MPI_VERSION >= 4
66# define MADNESS_MPI_HAS_LARGE_COUNT 1
67#endif
68
69
70#if MADNESS_MPI_THREAD_LEVEL == MPI_THREAD_SERIALIZED
71# define MADNESS_SERIALIZES_MPI
72#endif
73
74
75
78#include <atomic>
79#include <iostream>
80#include <limits>
81#include <csignal>
82#include <cstdlib>
83#include <cstring>
84#include <memory>
85#include <sstream>
86
87#define MADNESS_MPI_TEST(condition) \
88 { \
89 int mpi_error_code = condition; \
90 if(mpi_error_code != MPI_SUCCESS) { \
91 char s[MPI_MAX_ERROR_STRING]; \
92 s[0] = '\0'; \
93 int len = 0; \
94 MPI_Error_string(mpi_error_code, s, &len); \
95 std::cout<< "MPI ERROR in " << __FUNCTION__ << " in " << __FILE__ << " at line " << __LINE__ << " code " << mpi_error_code << " err string " << s << "\n"; \
96 throw ::SafeMPI::Exception(mpi_error_code); \
97 } \
98 }
99
100namespace SafeMPI {
101
102 extern madness::SCALABLE_MUTEX_TYPE charon; // Inside safempi.cc
103#ifdef MADNESS_SERIALIZES_MPI
104#define SAFE_MPI_GLOBAL_MUTEX madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
105#else
106#define SAFE_MPI_GLOBAL_MUTEX
107#endif
108
109 /// tags in [1,999] ... allocated once by unique_reserved_tag
110 ///
111 /// tags in [1000,1023] ... statically assigned here
112 ///
113 /// tags in [1024,4095] ... allocated round-robin by unique_tag
114 ///
115 /// tags in [4096,8191] ... reserved for huge msg exchange by RMI
116 ///
117 /// tags in [8192,MPI::TAG_UB] ... not used/managed by madness
118
119 static const int RMI_TAG = 1023;
120 static const int MPIAR_TAG = 1001;
121 static const int DEFAULT_SEND_RECV_TAG = 1000;
122
123 // Forward declarations
124 class Intracomm;
125 extern Intracomm COMM_WORLD;
126 inline int Finalize();
127
128 /// Check MPI initialization status
129
130 /// \return \c true if MPI has been initialized, \c false otherwise.
131 inline bool Is_initialized() {
132 int initialized = 0;
133 MPI_Initialized(&initialized);
134 return (initialized != 0);
135 }
136
137 /// Check MPI finalization status
138
139 /// \return \c true if MPI has been finalized, \c false otherwise.
140 inline bool Is_finalized() {
141 int flag = 0;
142 MPI_Finalized(&flag);
143 return flag != 0;
144 }
145
146 namespace detail {
147 /// Initialize SafeMPI::COMM_WORLD
148 inline void init_comm_world();
149
150 inline void print_mpi_error(const int rc, const char* function,
151 const int line, const char* file)
152 {
153 int len = 0;
154 char error_string[MPI_MAX_ERROR_STRING];
155 MPI_Error_string(rc, error_string, &len);
156 std::cerr << "!!! MPI ERROR (" << rc << ") in " << function <<
157 " at " << file << "(" << line << "): " << error_string << "\n";
158 }
159
160 } // namespace detail
161
162 /// SafeMPI exception object
163
164 /// This exception is thrown whenever an MPI error occurs.
165 class Exception : public std::exception {
166 private:
169 public:
170
171 Exception(const int mpi_error) throw() {
172 int len = 0;
173 if(MPI_Error_string(mpi_error, mpi_error_string_, &len) != MPI_SUCCESS)
174 std::strncpy(mpi_error_string_, "UNKNOWN MPI ERROR!", MPI_MAX_ERROR_STRING);
175 }
176
177 Exception(const int mpi_error, const int nstatuses,
178 const int* indices,
179 MPI_Status* const statuses) noexcept {
180 try {
181 if (mpi_error == MPI_ERR_IN_STATUS) {
182 std::ostringstream oss;
183 for(auto s=0; s!=nstatuses; ++s) {
184 int len = 0;
185 auto status_error = statuses[s].MPI_ERROR;
186 if (status_error != MPI_SUCCESS) {
187 oss << "request " << indices[s] << ":";
188 if (MPI_Error_string(status_error, mpi_error_string_, &len) != MPI_SUCCESS)
189 oss << " unknown error!" << std::endl;
190 else
191 oss << mpi_error_string_ << std::endl;
192 }
193 }
194 mpi_statuses_error_string_ = oss.str();
195 }
196 }
197 catch (...) {}
198
199 int len = 0;
200 if(MPI_Error_string(mpi_error, mpi_error_string_, &len) != MPI_SUCCESS)
201 std::strncpy(mpi_error_string_, "UNKNOWN MPI ERROR!", MPI_MAX_ERROR_STRING);
202 }
203
204 Exception(const Exception& other) throw() {
205 std::strncpy(mpi_error_string_, other.mpi_error_string_, MPI_MAX_ERROR_STRING);
206 try {
207 mpi_statuses_error_string_ = other.mpi_statuses_error_string_;
208 } catch(...) { mpi_statuses_error_string_.clear(); }
209 }
210
213 try {
215 } catch(...) { mpi_statuses_error_string_.clear(); }
216 return *this;
217 }
218
219 virtual ~Exception() throw() { }
220
221 virtual const char* what() const throw() { return mpi_error_string_; }
222 bool can_elaborate() const noexcept {
223 return !mpi_statuses_error_string_.empty();
224 }
225 const char* elaborate() const noexcept {
226 return mpi_statuses_error_string_.c_str();
227 }
228
229 friend std::ostream& operator<<(std::ostream& os, const Exception& e) {
230 os << e.what();
231 if (e.can_elaborate()) {
232 os << e.elaborate();
233 }
234 return os;
235 }
236 }; // class Exception
237
238
239 class Status {
240 private:
242
243 public:
244 // Constructors
245 Status(void) : status_() { }
246 Status(const Status &other) : status_(other.status_) { }
247 Status(MPI_Status other) : status_(other) { }
248
249 // Assignment operators
250 Status& operator=(const Status &other) {
251 status_ = other.status_;
252 return *this;
253 }
254
256 status_ = other;
257 return *this;
258 }
259
260 // C/C++ cast and assignment
261 operator MPI_Status*() { return &status_; }
262
263 operator MPI_Status() const { return status_; }
264
265// bool Is_cancelled() const {
266// int flag = 0;
267// MADNESS_MPI_TEST(MPI_Test_cancelled(const_cast<MPI_Status*>(&status_), &flag));
268// return flag != 0;
269// }
270//
271// int Get_elements(const MPI_Datatype datatype) const {
272// int elements = 0;
273// MADNESS_MPI_TEST(MPI_Get_elements(const_cast<MPI_Status*>(&status_), datatype, &elements));
274// return elements;
275// }
276
277 int Get_count(const MPI_Datatype datatype) const {
278 int count = 0;
279 MADNESS_MPI_TEST(MPI_Get_count(const_cast<MPI_Status*>(&status_), datatype, &count));
280 return count;
281 }
282
283 /// The size of a received message of bytes, which may exceed \c INT_MAX
284
285 /// \return the number of bytes received, or -1 if MPI cannot tell: without the
286 /// large-count functions of MPI 4 ( \c MPI_Get_count_c ) a count above \c INT_MAX
287 /// is not reported
288 long long Get_byte_count() const {
289#ifdef MADNESS_MPI_HAS_LARGE_COUNT
290 MPI_Count count = 0;
291 MADNESS_MPI_TEST(MPI_Get_count_c(const_cast<MPI_Status*>(&status_), MPI_BYTE, &count));
292#else
293 int count = 0;
294 MADNESS_MPI_TEST(MPI_Get_count(const_cast<MPI_Status*>(&status_), MPI_BYTE, &count));
295#endif
296 return count == MPI_UNDEFINED ? -1ll : static_cast<long long>(count);
297 }
298
299// void Set_cancelled(bool flag) {
300// MADNESS_MPI_TEST(MPI_Status_set_cancelled(&status_, flag));
301// }
302//
303// void Set_elements( const MPI_Datatype &v2, int v3 ) {
304// MADNESS_MPI_TEST(MPI_Status_set_elements(&status_, v2, v3 ));
305// }
306
307 int Get_source() const { return status_.MPI_SOURCE; }
308
309 int Get_tag() const { return status_.MPI_TAG; }
310
311 int Get_error() const { return status_.MPI_ERROR; }
312
313 void Set_source(int source) { status_.MPI_SOURCE = source; }
314
315 void Set_tag(int tag) { status_.MPI_TAG = tag; }
316
317 void Set_error(int error) { status_.MPI_ERROR = error; }
318 }; // class Status
319
320 class Request {
321 // Note: This class was previously derived from MPI::Request, but this
322 // was changed with the removal of the MPI C++ bindings. Now this class
323 // only implements the minimum functionality required by MADNESS. Feel
324 // free to add more functionality as needed.
325
326 private:
328
329 public:
330
331 // Constructors
333 Request(MPI_Request other) : request_(other) { }
334 Request(const Request& other) : request_(other.request_) { }
335
336 // Assignment operators
337 Request& operator=(const Request &other) {
338 request_ = other.request_;
339 return *this;
340 }
341
343 request_ = other;
344 return *this;
345 }
346
347 // logical
348 bool operator==(const Request &other) { return (request_ == other.request_); }
349 bool operator!=(const Request &other) { return (request_ != other.request_); }
350
351 // C/C++ cast and assignment
352 operator MPI_Request*() { return &request_; }
353 operator MPI_Request() const { return request_; }
354
355 static bool Testany(int count, Request* requests, int& index, Status& status) {
356 MADNESS_ASSERT(requests != nullptr);
357 int flag;
358 std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[count]);
359
360 // Copy requests to an array that can be used by MPI
361 for(int i = 0; i < count; ++i)
362 mpi_requests[i] = requests[i].request_;
363 {
365 MADNESS_MPI_TEST(MPI_Testany(count, mpi_requests.get(), &index, &flag, status));
366 }
367 // Copy results from MPI back to the original array
368 for(int i = 0; i < count; ++i)
369 requests[i].request_ = mpi_requests[i];
370 return flag != 0;
371 }
372
373 static bool Testany(int count, Request* requests, int& index) {
374 MADNESS_ASSERT(requests != nullptr);
375 int flag;
376 std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[count]);
377
378 // Copy requests to an array that can be used by MPI
379 for(int i = 0; i < count; ++i)
380 mpi_requests[i] = requests[i].request_;
381 {
383 MADNESS_MPI_TEST(MPI_Testany(count, mpi_requests.get(), &index, &flag, MPI_STATUS_IGNORE));
384 }
385 // Copy results from MPI back to the original array
386 for(int i = 0; i < count; ++i)
387 requests[i] = mpi_requests[i];
388 return flag != 0;
389 }
390
391 static int Testsome(int incount, Request* requests, int* indices, Status* statuses) {
392 MADNESS_ASSERT(requests != nullptr);
393 MADNESS_ASSERT(indices != nullptr);
394 MADNESS_ASSERT(statuses != nullptr);
395
396 int outcount = 0;
397 std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[incount]);
398 std::unique_ptr<MPI_Status[]> mpi_statuses(new MPI_Status[incount]);
399 for(int i = 0; i < incount; ++i)
400 mpi_requests[i] = requests[i].request_;
401 {
403 { // print out the status vars for the failed requests
404 auto mpi_error_code =
405 MPI_Testsome(incount, mpi_requests.get(), &outcount,
406 indices, mpi_statuses.get());
407 if (mpi_error_code != MPI_SUCCESS) {
408 throw ::SafeMPI::Exception(mpi_error_code, outcount, indices, mpi_statuses.get());
409 }
410 }
411 }
412 for(int i = 0; i < incount; ++i) {
413 requests[i] = mpi_requests[i];
414 statuses[i] = mpi_statuses[i];
415 }
416 return outcount;
417 }
418
419 static int Testsome(int incount, Request* requests, int* indices) {
420 int outcount = 0;
421 std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[incount]);
422 for(int i = 0; i < incount; ++i)
423 mpi_requests[i] = requests[i].request_;
424 {
426 MADNESS_MPI_TEST( MPI_Testsome( incount, mpi_requests.get(), &outcount, indices, MPI_STATUSES_IGNORE));
427 }
428 for(int i = 0; i < incount; ++i)
429 requests[i] = mpi_requests[i];
430 return outcount;
431 }
432
433
435 int flag;
437 return flag != 0;
438 }
439
444
446 int flag;
448 return flag != 0;
449 }
450
451 bool Test() {
453 return Test_got_lock_already();
454 }
455 }; // class Request
456
457 /// Wrapper around MPI_Group. Has a shallow copy constructor. Usually deep copy is not needed, but can be created
458 /// via Group::Incl().
459 class Group {
460 public:
461 Group Incl(int n, const int* ranks) const {
462 // MPI <3 interface lacks explicit const sanitation
463 Group result(std::shared_ptr<Impl>(new Impl(*pimpl, n, const_cast<int*>(ranks))));
464 return result;
465 }
466
467 void Translate_ranks(int nproc, const int* ranks1, const Group& grp2, int* ranks2) const {
468 // MPI <3 interface lacks explicit const sanitation
471 const_cast<int*>(ranks1), grp2.pimpl->group, ranks2));
472 }
473
474 MPI_Group group() const {
476 return pimpl->group;
477 }
478
479 Group(const Group& other) : pimpl(other.pimpl) { }
480
481 private:
482
483 struct Impl {
485
488 }
489
490 Impl(const Impl& other, int n, const int* ranks) {
491 // MPI <3 interface lacks explicit const sanitation
493 const_cast<int*>(ranks), & group));
494 }
495
497 if(Is_initialized()) {
498 const int mpi_error_code = MPI_Group_free(&group);
499 if(mpi_error_code != MPI_SUCCESS)
501 "SafeMPI::Group::Impl::~Impl()", __LINE__, __FILE__);
502 }
503 }
504
505 }; // struct Impl
506
507 friend class Intracomm;
508
509 Group() : pimpl() { }
510
511 // only Intracomm will use this
512 Group(MPI_Comm comm) : pimpl(new Impl(comm)) { }
513
514 // only myself will use this
515 Group(const std::shared_ptr<Impl>& p) : pimpl(p) { }
516
517 std::shared_ptr<Impl> pimpl;
518 }; // class Group
519
520 /// The (count, datatype) pair that describes a buffer of bytes to MPI
521
522 /// This is how a buffer of more than \c INT_MAX bytes is sent and received where MPI
523 /// has no large-count functions (before MPI 4), see Intracomm::Isend_bytes() .
524 /// MPI counts are \c int , so a buffer of more than \c INT_MAX bytes cannot be
525 /// described as a count of \c MPI_BYTE . Such a buffer is described here as one element
526 /// of a derived datatype: whole chunks of bytes, then the bytes left over. The type
527 /// signature is that of the same number of \c MPI_BYTE , so the two ends of a message
528 /// need not agree on how they describe it. A derived datatype is freed by the
529 /// destructor; MPI lets a datatype be freed while a nonblocking operation that uses it
530 /// is pending, so the object need not outlive the request.
531 class ByteCount {
532 public:
533 /// \param nbyte the size of the buffer
534 /// \param chunk a buffer of up to this many bytes is described as a count of
535 /// \c MPI_BYTE , and a larger one as chunks of this many bytes plus the rest
536 /// (default: max_plain_count() ; 0, or a value above \c INT_MAX , means \c INT_MAX )
537 explicit ByteCount(std::size_t nbyte, std::size_t chunk = ByteCount::max_plain_count())
538 : count_(0), datatype_(MPI_BYTE), derived_(false) {
539 const std::size_t int_max = static_cast<std::size_t>(std::numeric_limits<int>::max());
540 if (chunk == 0 || chunk > int_max) chunk = int_max;
541 if (nbyte <= chunk) {
542 count_ = static_cast<int>(nbyte);
543 return;
544 }
545
546 const std::size_t nchunk = nbyte / chunk;
547 const std::size_t nleft = nbyte % chunk;
548 // a chunk count beyond int would take chunks of chunks; with the default chunk
549 // that is a buffer of 2^62 bytes
550 if (nchunk > int_max) {
551 std::cout << "SafeMPI::ByteCount: " << nbyte << " bytes in chunks of " << chunk
552 << " bytes: more chunks than an MPI count can hold\n";
553 throw ::SafeMPI::Exception(MPI_ERR_COUNT);
554 }
556 MPI_Datatype chunk_type;
557 MADNESS_MPI_TEST(MPI_Type_contiguous(static_cast<int>(chunk), MPI_BYTE, &chunk_type));
558 int blocklengths[2] = {static_cast<int>(nchunk), static_cast<int>(nleft)};
559 MPI_Aint displacements[2] = {0, static_cast<MPI_Aint>(nchunk * chunk)};
560 MPI_Datatype types[2] = {chunk_type, MPI_BYTE};
561 MPI_Datatype whole;
562 int status = MPI_Type_create_struct(nleft ? 2 : 1, blocklengths, displacements, types, &whole);
563 // the struct keeps what it needs of the chunk type, which is freed either way
564 MPI_Type_free(&chunk_type);
565 if (status == MPI_SUCCESS) {
566 status = MPI_Type_commit(&whole);
567 if (status != MPI_SUCCESS) MPI_Type_free(&whole);
568 }
570 datatype_ = whole;
571 count_ = 1;
572 derived_ = true;
573 }
574
575 ByteCount(const ByteCount&) = delete;
576 ByteCount& operator=(const ByteCount&) = delete;
577
579 if (derived_) {
582 }
583 }
584
585 /// \return the count to give MPI together with datatype()
586 int count() const { return count_; }
587 MPI_Datatype datatype() const { return datatype_; }
588 /// \return whether the buffer was too large for a count of \c MPI_BYTE
589 bool derived() const { return derived_; }
590
591 /// \return the largest buffer that is described as a count of \c MPI_BYTE
592 static std::size_t max_plain_count() { return max_plain_count_accessor(); }
593
594 /// Sets max_plain_count()
595
596 /// This exists for tests, which lower it to take the derived-datatype path with
597 /// small buffers; nothing else has a reason to call it. The two ends of a message
598 /// may use different values, and a change while messages are in flight is safe:
599 /// each ByteCount reads the value once.
600 /// \param n the new value; 0, or a value above \c INT_MAX , restores the default
601 static void set_max_plain_count(std::size_t n) {
602 const std::size_t int_max = static_cast<std::size_t>(std::numeric_limits<int>::max());
603 max_plain_count_accessor() = (n == 0 || n > int_max) ? int_max : n;
604 }
605
606 /// \return whether this MPI has the large-count functions of MPI 4
607 /// ( \c MPI_Isend_c and the like), which take a count of type \c MPI_Count
608 static constexpr bool have_large_count() {
609#ifdef MADNESS_MPI_HAS_LARGE_COUNT
610 return true;
611#else
612 return false;
613#endif
614 }
615
616 /// \return whether buffers above max_plain_count() are given to the large-count
617 /// functions of MPI rather than described by a derived datatype; true by default
618 /// if have_large_count() , and always false otherwise
620
621 /// Sets use_large_count()
622
623 /// Like set_max_plain_count() this exists for tests, which turn it off to take
624 /// the derived-datatype path with an MPI that has large counts. The two ends of a
625 /// message need not agree.
626 static void set_use_large_count(bool flag) { use_large_count_accessor() = flag; }
627
628 private:
629 static std::atomic<bool>& use_large_count_accessor() {
630 static std::atomic<bool> value{true};
631 return value;
632 }
633
634 static std::atomic<std::size_t>& max_plain_count_accessor() {
635 static std::atomic<std::size_t> value{static_cast<std::size_t>(std::numeric_limits<int>::max())};
636 return value;
637 }
638
642 }; // class ByteCount
643
644 /// Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy
645 class Intracomm {
646
647 static bool Comm_compare(const MPI_Comm& comm1, const MPI_Comm& comm2) {
648 int compare_result;
649 const int result = MPI_Comm_compare(comm1, comm2, &compare_result);
650 return ((result == MPI_SUCCESS) && (compare_result == MPI_IDENT));
651 }
652
653 struct Impl {
655 int me;
657 bool owner;
658
659 int utag; // Only used by main thread so no volatile or mutex needed
660 int urtag;// Ditto
661
662 Impl(const MPI_Comm& c, int m, int n, bool o) :
663 comm(c), me(m), numproc(n), owner(o), utag(1024), urtag(1)
665
671
672 /// Returns a unique tag for temporary use (1023<tag<=4095)
673
674 /// These tags are intended for one time use to avoid tag
675 /// collisions with other messages around the same time period.
676 /// It simply increments/wraps a counter and returns the next
677 /// legal value.
678 ///
679 /// So that send and receiver agree on the tag all processes
680 /// need to call this routine in the same sequence.
682 // RJH removed mutex since ordering requirement across processes means
683 // there can never be any thread contention.
684 // Cannot use MPI mutex for anything else!
685 // It will preprocess to nothing for MPI_THREAD_MULTIPLE!
686 //madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
687 int result = utag++;
688 if (utag >= 4095) utag = 1024;
689 return result;
690 }
691
692 /// \return the period of repeat of unique tags produces by unique_tag()
693 static int unique_tag_period() {
694 const auto min_tag_value = 1024;
695 const auto max_tag_value = 4094;
696 return max_tag_value - min_tag_value + 1;
697 }
698
699 /// Returns a unique tag reserved for long-term use (0<tag<1000)
700
701 /// Get a tag from this routine for long-term/repeated use.
702 ///
703 /// Tags in [1000,1023] are statically assigned.
705 // RJH removed mutex since ordering requirement across processes means
706 // Cannot use MPI mutex for anything else!
707 // It will preprocess to nothing for MPI_THREAD_MULTIPLE!
708 // madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
709 int result = urtag++;
710 if (result >= 1000) MADNESS_EXCEPTION( "too many reserved tags in use" , result );
711 return result;
712 }
713
714 };
715 std::shared_ptr<Impl> pimpl;
716
718 friend int Finalize();
719
720 // For internal use only. Do not try to call this constructor. It is
721 // only used to construct Intarcomm in Create().
722 Intracomm(const std::shared_ptr<Impl>& i) : pimpl(i) { }
723
724 // Not allowed
726
727 // makes an uninitialized ptr
728 Intracomm() : pimpl(nullptr) {}
729
730 public:
731 struct WorldInitObject;
732
733 // For internal use only. Do not try to call this constructor. It is
734 // only used to construct COMM_WORLD.
735 Intracomm(const WorldInitObject&);
736
737 explicit Intracomm(const MPI_Comm& comm, bool take_ownership_of_comm = true) :
738 pimpl()
739 {
741 int rank = -1, size = -1;
742 MADNESS_MPI_TEST(MPI_Comm_rank(comm, &rank));
743 MADNESS_MPI_TEST(MPI_Comm_size(comm, &size));
744 take_ownership_of_comm =
745 take_ownership_of_comm && (! Comm_compare(comm, MPI_COMM_WORLD));
746 pimpl.reset(new Impl(comm, rank, size, take_ownership_of_comm));
747 }
748
749 Intracomm(const Intracomm& other) : pimpl(other.pimpl) { }
750
752
753 /**
754 * This collective operation creates a new \c Intracomm from an
755 * \c Intracomm::Group object. Must be called by all processes that
756 * belong to this communicator, but not all must use the same \c group .
757 * Thus this \c Intracomm can be partitioned into several \c Intracomm
758 * objects with one call.
759 *
760 * @param group Intracomm::Group describing the Intracomm object to be
761 * created (\c Intracomm::Get_group() and \c Intracomm::Group::Incl() )
762 * @return a new Intracomm object
763 */
764 Intracomm Create(Group group) const {
767 MPI_Comm group_comm;
768 MADNESS_MPI_TEST(MPI_Comm_create(pimpl->comm, group.group(), &group_comm));
769 int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me));
770 int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc));
771 return Intracomm(std::shared_ptr<Impl>(new Impl(group_comm, me, nproc, true)));
772 }
773
774 static const int UNDEFINED_COLOR = MPI_UNDEFINED;
775 /**
776 * This collective operation creates a new \c Intracomm using
777 * the MPI_Comm_split. Must be called by all processes that
778 * belong to this communicator. Each caller must provide Color of the new Intracomm
779 * and Key (this controls the rank within the new Intracomm;
780 * ties are broken by the rank in this Intracomm).
781 *
782 * @param Color Specifies the new Intracomm that the calling process is to be assigned to.
783 * The value of color must be non-negative. If Color=UNDEFINED_COLOR then
784 * an uninitialized Intracomm object will be produced.
785 * @param Key The relative rank of the calling process in the group of the new Intracomm.
786 * If omitted, each communicator's ranks will be determined by
787 * the rank in the host communicator.
788 * @return a new Intracomm object
789 */
790 Intracomm Split(int Color, int Key = 0) const {
793 MPI_Comm group_comm;
794 MADNESS_MPI_TEST(MPI_Comm_split(pimpl->comm, Color, Key, &group_comm));
795 if (group_comm != MPI_COMM_NULL) {
796 int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me));
797 int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc));
798 return Intracomm(std::shared_ptr<Impl>(new Impl(group_comm, me, nproc, true)));
799 }
800 else
801 return Intracomm();
802 }
803
806 /**
807 * This collective operation creates a new \c Intracomm using
808 * the MPI_Comm_split_type. Must be called by all processes that
809 * belong to this communicator. Each caller must provide the split type
810 * and Key (this controls the rank within the new Intracomm;
811 * ties are broken by the rank in this Intracomm).
812 *
813 * @param Type Controls how this Intracomm will be split.
814 * The value can only be UNDEFINED_SPLIT_TYPE or SHARED_SPLIT_TYPE.
815 * If Type=UNDEFINED_SPLIT_TYPE then
816 * an uninitialized Intracomm object will be produced.
817 * @param Key The relative rank of the calling process in the group of the new Intracomm.
818 * If omitted, each communicator's ranks will be determined by
819 * the rank in the host communicator.
820 * @return a new Intracomm object
821 */
822 Intracomm Split_type(int Type, int Key = 0) const {
825 MPI_Comm group_comm;
826 MPI_Info info;
827 MPI_Info_create(&info);
828 MADNESS_MPI_TEST(MPI_Comm_split_type(pimpl->comm, Type, Key, info, &group_comm));
829 MPI_Info_free(&info);
830 if (group_comm != MPI_COMM_NULL) {
831 int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me));
832 int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc));
833 return Intracomm(std::shared_ptr<Impl>(new Impl(group_comm, me, nproc, true)));
834 }
835 else
836 return Intracomm();
837 }
838
839 /**
840 * Clones this Intracomm object
841 *
842 * @return a (deep) copy of this Intracomm object
843 */
844 Intracomm Clone() const {
845 return Create(this->Get_group());
846 }
847
848 bool operator==(const Intracomm& other) const {
849 return (pimpl == other.pimpl) || ((pimpl && other.pimpl) &&
850 Comm_compare(pimpl->comm, other.pimpl->comm));
851 }
852
853 /**
854 * This local operation returns the Intracomm::Group object corresponding to this intracommunicator
855 * @return the Intracomm::Group object corresponding to this intracommunicator
856 */
857 Group Get_group() const {
860 Group group(pimpl->comm);
861 return group;
862 }
863
866 return pimpl->comm;
867 }
868
869 int Get_rank() const {
871 return pimpl->me;
872 }
873
874 int Get_size() const {
876 return pimpl->numproc;
877 }
878
879 Request Isend(const void* buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const {
882 Request request;
883 MADNESS_MPI_TEST(MPI_Isend(const_cast<void*>(buf), count, datatype, dest,tag, pimpl->comm, request));
884 return request;
885 }
886
887 Request Issend(const void* buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const {
890 Request request;
891 MADNESS_MPI_TEST(MPI_Issend(const_cast<void*>(buf), count, datatype, dest,tag, pimpl->comm, request));
892 return request;
893 }
894
895 Request Irecv(void* buf, const int count, const MPI_Datatype datatype, const int src, const int tag) const {
898 Request request;
899 MADNESS_MPI_TEST(MPI_Irecv(buf, count, datatype, src, tag, pimpl->comm, request));
900 return request;
901 }
902
903 /// Nonblocking send of \c nbyte bytes; unlike Isend() , \c nbyte may exceed \c INT_MAX
904
905 /// Up to ByteCount::max_plain_count() bytes ( \c INT_MAX unless a test lowered it)
906 /// are sent as Isend() sends them. More are given to the large-count function of
907 /// MPI 4 ( \c MPI_Isend_c ) where MPI has it, and are otherwise described by a
908 /// derived datatype (see ByteCount ).
909 /// \param sync if true, the send is synchronous ( \c MPI_Issend )
910 Request Isend_bytes(const void* buf, const std::size_t nbyte, const int dest, const int tag, const bool sync = false) const {
911 const std::size_t max_plain = ByteCount::max_plain_count();
912#ifdef MADNESS_MPI_HAS_LARGE_COUNT
913 if (nbyte > max_plain && ByteCount::use_large_count()) {
916 Request request;
917 const MPI_Count count = static_cast<MPI_Count>(nbyte);
918 MADNESS_MPI_TEST(sync ? MPI_Issend_c(buf, count, MPI_BYTE, dest, tag, pimpl->comm, request)
919 : MPI_Isend_c(buf, count, MPI_BYTE, dest, tag, pimpl->comm, request));
920 return request;
921 }
922#endif
923 const ByteCount bc(nbyte, max_plain);
924 return sync ? Issend(buf, bc.count(), bc.datatype(), dest, tag)
925 : Isend(buf, bc.count(), bc.datatype(), dest, tag);
926 }
927
928 /// Nonblocking receive of \c nbyte bytes; unlike Irecv() , \c nbyte may exceed \c INT_MAX
929
930 /// The large-count function of MPI 4 ( \c MPI_Irecv_c ) or a derived datatype is
931 /// used as in Isend_bytes() ; the sender need not have made the same choice.
932 /// \note the size of the message received is Status::Get_byte_count() , which
933 /// cannot report more than \c INT_MAX bytes without MPI 4; the receiver should
934 /// know the size
935 Request Irecv_bytes(void* buf, const std::size_t nbyte, const int src, const int tag) const {
936 const std::size_t max_plain = ByteCount::max_plain_count();
937#ifdef MADNESS_MPI_HAS_LARGE_COUNT
938 if (nbyte > max_plain && ByteCount::use_large_count()) {
941 Request request;
942 MADNESS_MPI_TEST(MPI_Irecv_c(buf, static_cast<MPI_Count>(nbyte), MPI_BYTE, src, tag, pimpl->comm, request));
943 return request;
944 }
945#endif
946 const ByteCount bc(nbyte, max_plain);
947 return Irecv(buf, bc.count(), bc.datatype(), src, tag);
948 }
949
950 void Send(const void* buf, const int count, const MPI_Datatype datatype, int dest, int tag) const {
953 MADNESS_MPI_TEST(MPI_Ssend(const_cast<void*>(buf), count, datatype, dest, tag, pimpl->comm));
954 }
955
956#ifdef MADNESS_USE_BSEND_ACKS
957 void Bsend(const void* buf, size_t count, const MPI_Datatype datatype, int dest, int tag) const {
960 if (count>10 || datatype!=MPI_BYTE) MADNESS_EXCEPTION("Bsend: this protocol is only for 1-byte acks", count );
961 MADNESS_MPI_TEST(MPI_Bsend(const_cast<void*>(buf), count, datatype, dest, tag, pimpl->comm));
962 }
963#endif // MADNESS_USE_BSEND_ACKS
964
965 void Recv(void* buf, const int count, const MPI_Datatype datatype, const int source, const int tag, MPI_Status& status) const {
968 MADNESS_MPI_TEST(MPI_Recv(buf, count, datatype, source, tag, pimpl->comm, &status));
969 }
970
971 void Recv(void* buf, const int count, const MPI_Datatype datatype, const int source, const int tag) const {
974 MADNESS_MPI_TEST(MPI_Recv(buf, count, datatype, source, tag, pimpl->comm, MPI_STATUS_IGNORE));
975 }
976
977 void Bcast(void* buf, size_t count, const MPI_Datatype datatype, const int root) const {
980 MADNESS_MPI_TEST(MPI_Bcast(buf, count, datatype, root, pimpl->comm));
981 }
982
983 void Reduce(const void* sendbuf, void* recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op, const int root) const {
986 MADNESS_MPI_TEST(MPI_Reduce(const_cast<void*>(sendbuf), recvbuf, count, datatype, op, root, pimpl->comm));
987 }
988
989 void Allreduce(const void* sendbuf, void* recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const {
992 MADNESS_MPI_TEST(MPI_Allreduce(const_cast<void*>(sendbuf), recvbuf, count, datatype, op, pimpl->comm));
993 }
994 bool Get_attr(int key, void* value) const {
996 int flag = 0;
998 MADNESS_MPI_TEST(MPI_Comm_get_attr(pimpl->comm, key, value, &flag));
999 return flag != 0;
1000 }
1001
1002 void Abort(int code=1) const {
1003 // if SIGABRT has a handler, call std::abort to allow it be caught,
1004 // else call MPI_Abort which does not seem to call abort at all,
1005 // instead sends SIGTERM followed by SIGKILL
1006 // if have a custom signal handler for SIGABRT (i.e. we are running under a
1007 // debugger) then call abort()
1008 struct sigaction sa;
1009 auto rc = sigaction(SIGABRT, NULL, &sa);
1010 if (rc == 0 && sa.sa_handler != SIG_DFL) {
1011 std::abort();
1012 } else {
1013 MPI_Abort(pimpl->comm, code);
1014 }
1015 }
1016
1017 void Barrier() const {
1021 }
1022
1023 /// Returns a unique tag for temporary use (1023<tag<4095)
1024
1025 /// These tags are intended for one time use to avoid tag
1026 /// collisions with other messages around the same time period.
1027 /// It simply increments/wraps a counter and returns the next
1028 /// legal value.
1029 ///
1030 /// So that send and receiver agree on the tag all processes
1031 /// need to call this routine in the same sequence.
1034 return pimpl->unique_tag();
1035 }
1036
1037 /// \return the period of repeat of unique tags produces by unique_tag()
1038 static int unique_tag_period() {
1039 return Impl::unique_tag_period();
1040 }
1041
1042 /// Returns a unique tag reserved for long-term use (0<tag<1000)
1043
1044 /// Get a tag from this routine for long-term/repeated use.
1045 ///
1046 /// Tags in [1000,1023] are statically assigned.
1049 return pimpl->unique_reserved_tag();
1050 }
1051
1052 /// Construct info about a binary tree with given root
1053
1054 /// Constructs a binary tree spanning the communicator with
1055 /// process root as the root of the tree. Returns the logical
1056 /// parent and children in the tree of the calling process. If
1057 /// there is no parent/child the value -1 will be set.
1058 void binary_tree_info(int root, int& parent, int& child0, int& child1);
1059
1060 }; // class Intracomm
1061
1062 namespace detail {
1063
1064 /// Initialize SafeMPI::COMM_WORLD
1065 inline void init_comm_world() {
1066 MADNESS_MPI_TEST(MPI_Comm_rank(COMM_WORLD.pimpl->comm, & COMM_WORLD.pimpl->me));
1067 MADNESS_MPI_TEST(MPI_Comm_size(COMM_WORLD.pimpl->comm, & COMM_WORLD.pimpl->numproc));
1069 }
1070
1071 } // namespace detail
1072
1073
1074 /// Analogous to MPI_Init_thread
1075
1076 /// \param argc the number of arguments in argv
1077 /// \param argv the vector of command-line arguments
1078 /// \param requested the desired thread level
1079 /// \return provided thread level
1080 inline int Init_thread(int & argc, char **& argv, int requested) {
1081 int provided = 0;
1082 MADNESS_MPI_TEST(MPI_Init_thread(&argc, &argv, requested, &provided));
1084 return provided;
1085 }
1086
1087 /// Analogous to MPI_Init_thread
1088
1089 /// \param requested the desired thread level
1090 /// \return provided thread level
1091 inline int Init_thread(int requested) {
1092 int argc = 0;
1093 char** argv = nullptr;
1094 return SafeMPI::Init_thread(argc, argv, requested);
1095 }
1096
1097 /// Analogous to MPI_Init
1098
1099 /// \param argc The number of arguments in argv
1100 /// \param argv The vector of command-line arguments
1101 inline void Init(int &argc, char **&argv) {
1102 MADNESS_MPI_TEST(MPI_Init(&argc, &argv));
1104 }
1105
1106 /// Analogous to MPI_Init
1107 inline void Init() {
1108 int argc = 0;
1109 char** argv = nullptr;
1110 SafeMPI::Init(argc,argv);
1111 }
1112
1113 /// Analogous to MPI_Finalize
1114
1115 /// This returns status rather than throw an
1116 /// exception upon failure because this is a "destructor", and throwing from
1117 /// destructors is evil.
1118 /// \return 0 if successful, nonzero otherwise (see MPI_Finalize() for the
1119 /// return codes).
1120 inline int Finalize() {
1121 SafeMPI::COMM_WORLD.pimpl.reset();
1122 const int result = MPI_Finalize();
1123 return result;
1124 }
1125
1126 /// Analogous to MPI_Query_thread
1127
1128 /// \return the MPI thread level provided by SafeMPI::Init_thread()
1129 inline int Query_thread() {
1130 int provided;
1132 return provided;
1133 }
1134
1135 /// Wall time
1136
1137 /// \return The current wall time
1138 inline double Wtime() { return MPI_Wtime(); }
1139
1140 /// Set buffer for \c Bsend .
1141
1142 /// \param buffer The buffer to be used by Bsend
1143 /// \param size The size of the buffer in Bytes
1144 inline void Attach_buffer(void* buffer, int size) {
1145 MADNESS_MPI_TEST(MPI_Buffer_attach(buffer, size));
1146 }
1147
1148 /// Unset the \c Bsend buffer.
1149
1150 /// \param[out] buffer The buffer that was used by Bsend
1151 inline int Detach_buffer(void *&buffer) {
1152 int size = 0;
1153 MPI_Buffer_detach(&buffer, &size);
1154 return size;
1155 }
1156
1157 /// Analogous to MPI_Op_create
1158 inline MPI_Op Op_create(MPI_User_function *user_fn, int commute) {
1159 MPI_Op result;
1160 MADNESS_MPI_TEST(MPI_Op_create(user_fn, commute, &result));
1161 return result;
1162 }
1163
1164 /// Analogous to MPI_Op_free
1165 inline void Op_free(MPI_Op op) {
1167 }
1168
1169} // namespace SafeMPI
1170
1171#endif // MADNESS_WORLD_SAFEMPI_H__INCLUDED
The (count, datatype) pair that describes a buffer of bytes to MPI.
Definition safempi.h:531
ByteCount(const ByteCount &)=delete
bool derived() const
Definition safempi.h:589
int count_
Definition safempi.h:639
int count() const
Definition safempi.h:586
static std::size_t max_plain_count()
Definition safempi.h:592
MPI_Datatype datatype_
Definition safempi.h:640
static bool use_large_count()
Definition safempi.h:619
static void set_use_large_count(bool flag)
Sets use_large_count().
Definition safempi.h:626
MPI_Datatype datatype() const
Definition safempi.h:587
ByteCount & operator=(const ByteCount &)=delete
static std::atomic< bool > & use_large_count_accessor()
Definition safempi.h:629
~ByteCount()
Definition safempi.h:578
bool derived_
Definition safempi.h:641
static std::atomic< std::size_t > & max_plain_count_accessor()
Definition safempi.h:634
static constexpr bool have_large_count()
Definition safempi.h:608
ByteCount(std::size_t nbyte, std::size_t chunk=ByteCount::max_plain_count())
Definition safempi.h:537
static void set_max_plain_count(std::size_t n)
Sets max_plain_count().
Definition safempi.h:601
std::string mpi_statuses_error_string_
Definition safempi.h:168
bool can_elaborate() const noexcept
Definition safempi.h:222
Exception(const int mpi_error)
Definition safempi.h:171
char mpi_error_string_[MPI_MAX_ERROR_STRING]
Definition safempi.h:167
Exception & operator=(const Exception &other)
Definition safempi.h:211
friend std::ostream & operator<<(std::ostream &os, const Exception &e)
Definition safempi.h:229
virtual const char * what() const
Definition safempi.h:221
Exception(const int mpi_error, const int nstatuses, const int *indices, MPI_Status *const statuses) noexcept
Definition safempi.h:177
virtual ~Exception()
Definition safempi.h:219
const char * elaborate() const noexcept
Definition safempi.h:225
Exception(const Exception &other)
Definition safempi.h:204
Definition safempi.h:459
friend class Intracomm
Definition safempi.h:507
Group(const std::shared_ptr< Impl > &p)
Definition safempi.h:515
Group Incl(int n, const int *ranks) const
Definition safempi.h:461
MPI_Group group() const
Definition safempi.h:474
std::shared_ptr< Impl > pimpl
Definition safempi.h:517
Group(MPI_Comm comm)
Definition safempi.h:512
void Translate_ranks(int nproc, const int *ranks1, const Group &grp2, int *ranks2) const
Definition safempi.h:467
Group(const Group &other)
Definition safempi.h:479
Group()
Definition safempi.h:509
Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy.
Definition safempi.h:645
Intracomm(const std::shared_ptr< Impl > &i)
Definition safempi.h:722
Intracomm Clone() const
Definition safempi.h:844
Request Isend_bytes(const void *buf, const std::size_t nbyte, const int dest, const int tag, const bool sync=false) const
Nonblocking send of nbyte bytes; unlike Isend() , nbyte may exceed INT_MAX.
Definition safempi.h:910
void Allreduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const
Definition safempi.h:989
static bool Comm_compare(const MPI_Comm &comm1, const MPI_Comm &comm2)
Definition safempi.h:647
bool Get_attr(int key, void *value) const
Definition safempi.h:994
static const int SHARED_SPLIT_TYPE
Definition safempi.h:805
Intracomm Create(Group group) const
Definition safempi.h:764
Request Irecv_bytes(void *buf, const std::size_t nbyte, const int src, const int tag) const
Nonblocking receive of nbyte bytes; unlike Irecv() , nbyte may exceed INT_MAX.
Definition safempi.h:935
Request Irecv(void *buf, const int count, const MPI_Datatype datatype, const int src, const int tag) const
Definition safempi.h:895
void Bcast(void *buf, size_t count, const MPI_Datatype datatype, const int root) const
Definition safempi.h:977
MPI_Comm & Get_mpi_comm() const
Definition safempi.h:864
void Reduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op, const int root) const
Definition safempi.h:983
Request Isend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition safempi.h:879
friend int Finalize()
Analogous to MPI_Finalize.
Definition safempi.h:1120
void binary_tree_info(int root, int &parent, int &child0, int &child1)
Construct info about a binary tree with given root.
Definition safempi.cc:39
Intracomm & operator=(const Intracomm &other)
Intracomm Split_type(int Type, int Key=0) const
Definition safempi.h:822
int Get_rank() const
Definition safempi.h:869
static int unique_tag_period()
Definition safempi.h:1038
void Abort(int code=1) const
Definition safempi.h:1002
static const int UNDEFINED_SPLIT_TYPE
Definition safempi.h:804
Intracomm(const MPI_Comm &comm, bool take_ownership_of_comm=true)
Definition safempi.h:737
int unique_tag()
Returns a unique tag for temporary use (1023<tag<4095).
Definition safempi.h:1032
bool operator==(const Intracomm &other) const
Definition safempi.h:848
void Barrier() const
Definition safempi.h:1017
std::shared_ptr< Impl > pimpl
Definition safempi.h:715
Group Get_group() const
Definition safempi.h:857
void Send(const void *buf, const int count, const MPI_Datatype datatype, int dest, int tag) const
Definition safempi.h:950
Intracomm Split(int Color, int Key=0) const
Definition safempi.h:790
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000).
Definition safempi.h:1047
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag) const
Definition safempi.h:971
~Intracomm()
Definition safempi.h:751
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag, MPI_Status &status) const
Definition safempi.h:965
Intracomm(const Intracomm &other)
Definition safempi.h:749
int Get_size() const
Definition safempi.h:874
Intracomm()
Definition safempi.h:728
static const int UNDEFINED_COLOR
Definition safempi.h:774
Request Issend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition safempi.h:887
Definition safempi.h:320
bool operator!=(const Request &other)
Definition safempi.h:349
MPI_Request request_
Definition safempi.h:327
bool Test(MPI_Status &status)
Definition safempi.h:440
Request & operator=(const MPI_Request &other)
Definition safempi.h:342
bool Test_got_lock_already()
Definition safempi.h:445
static bool Testany(int count, Request *requests, int &index)
Definition safempi.h:373
static int Testsome(int incount, Request *requests, int *indices)
Definition safempi.h:419
Request()
Definition safempi.h:332
Request(MPI_Request other)
Definition safempi.h:333
static int Testsome(int incount, Request *requests, int *indices, Status *statuses)
Definition safempi.h:391
bool Test_got_lock_already(MPI_Status &status)
Definition safempi.h:434
static bool Testany(int count, Request *requests, int &index, Status &status)
Definition safempi.h:355
Request & operator=(const Request &other)
Definition safempi.h:337
bool operator==(const Request &other)
Definition safempi.h:348
Request(const Request &other)
Definition safempi.h:334
bool Test()
Definition safempi.h:451
Definition safempi.h:239
int Get_count(const MPI_Datatype datatype) const
Definition safempi.h:277
Status & operator=(const MPI_Status other)
Definition safempi.h:255
int Get_error() const
Definition safempi.h:311
Status & operator=(const Status &other)
Definition safempi.h:250
void Set_tag(int tag)
Definition safempi.h:315
Status(const Status &other)
Definition safempi.h:246
MPI_Status status_
Definition safempi.h:241
int Get_tag() const
Definition safempi.h:309
Status(MPI_Status other)
Definition safempi.h:247
int Get_source() const
Definition safempi.h:307
Status(void)
Definition safempi.h:245
void Set_source(int source)
Definition safempi.h:313
long long Get_byte_count() const
The size of a received message of bytes, which may exceed INT_MAX.
Definition safempi.h:288
void Set_error(int error)
Definition safempi.h:317
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition derivatives.cc:72
static double function(const coord_3d &r)
Normalized gaussian.
Definition functionio.cc:100
Tensor< double > op(const Tensor< double > &x)
Definition kain.cc:508
Macros and tools pertaining to the configuration of MADNESS.
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_ASSERT(condition)
Assert a condition that should be free of side-effects since in release builds this might be a no-op.
Definition madness_exception.h:134
void init_comm_world()
Initialize SafeMPI::COMM_WORLD.
Definition safempi.h:1065
void print_mpi_error(const int rc, const char *function, const int line, const char *file)
Definition safempi.h:150
Definition safempi.cc:35
Intracomm COMM_WORLD
Definition safempi.cc:67
void Init()
Analogous to MPI_Init.
Definition safempi.h:1107
static const int MPIAR_TAG
Definition safempi.h:120
int Init_thread(int &argc, char **&argv, int requested)
Analogous to MPI_Init_thread.
Definition safempi.h:1080
void Op_free(MPI_Op op)
Analogous to MPI_Op_free.
Definition safempi.h:1165
void Attach_buffer(void *buffer, int size)
Set buffer for Bsend .
Definition safempi.h:1144
madness::SCALABLE_MUTEX_TYPE charon
Definition safempi.cc:37
int Query_thread()
Analogous to MPI_Query_thread.
Definition safempi.h:1129
int Finalize()
Analogous to MPI_Finalize.
Definition safempi.h:1120
double Wtime()
Wall time.
Definition safempi.h:1138
bool Is_initialized()
Check MPI initialization status.
Definition safempi.h:131
static const int DEFAULT_SEND_RECV_TAG
Definition safempi.h:121
static const int RMI_TAG
Definition safempi.h:119
int Detach_buffer(void *&buffer)
Unset the Bsend buffer.
Definition safempi.h:1151
bool Is_finalized()
Check MPI finalization status.
Definition safempi.h:140
MPI_Op Op_create(MPI_User_function *user_fn, int commute)
Analogous to MPI_Op_create.
Definition safempi.h:1158
Definition potentialmanager.cc:41
Mutex SCALABLE_MUTEX_TYPE
Definition worldmutex.h:715
void error(const char *msg, int code)
Definition oldtest.cc:57
static const double c
Definition relops.cc:10
static const double m
Definition relops.cc:9
#define MADNESS_MPI_TEST(condition)
Definition safempi.h:87
#define SAFE_MPI_GLOBAL_MUTEX
Definition safempi.h:104
Definition test_dc.cc:47
Definition stubmpi.h:13
Definition safempi.h:483
Impl(MPI_Comm comm)
Definition safempi.h:486
MPI_Group group
Definition safempi.h:484
~Impl()
Definition safempi.h:496
Impl(const Impl &other, int n, const int *ranks)
Definition safempi.h:490
Definition safempi.h:653
int numproc
Definition safempi.h:656
int urtag
Definition safempi.h:660
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000).
Definition safempi.h:704
bool owner
Definition safempi.h:657
static int unique_tag_period()
Definition safempi.h:693
int me
Definition safempi.h:655
int utag
Definition safempi.h:659
MPI_Comm comm
Definition safempi.h:654
int unique_tag()
Returns a unique tag for temporary use (1023<tag<=4095).
Definition safempi.h:681
Impl(const MPI_Comm &c, int m, int n, bool o)
Definition safempi.h:662
~Impl()
Definition safempi.h:666
#define MPI_SUCCESS
Definition stubmpi.h:36
int MPI_Ssend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition stubmpi.h:219
#define MPI_ERRORS_RETURN
Definition stubmpi.h:41
int MPI_Type_contiguous(int, MPI_Datatype, MPI_Datatype *)
Definition stubmpi.h:94
int MPI_Buffer_detach(void *, int *)
Definition stubmpi.h:188
int MPI_Bcast(void *, int, MPI_Datatype, int, MPI_Comm)
Definition stubmpi.h:248
#define MPI_COMM_NULL
Definition stubmpi.h:51
int MPI_Error_string(int errorcode, char *string, int *resultlen)
Definition stubmpi.h:303
int MPI_Isend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition stubmpi.h:216
int MPI_Op
Definition stubmpi.h:143
int MPI_Request
Definition stubmpi.h:12
#define MPI_ERR_IN_STATUS
Definition stubmpi.h:40
void MPI_User_function(void *a, void *b, int *len, MPI_Datatype *)
Definition stubmpi.h:159
int MPI_Op_create(MPI_User_function *, int, MPI_Op *)
Definition stubmpi.h:329
std::ptrdiff_t MPI_Aint
Definition stubmpi.h:32
#define MPI_COMM_WORLD
Definition stubmpi.h:25
int MPI_Group
Definition stubmpi.h:11
int MPI_Type_free(MPI_Datatype *)
Definition stubmpi.h:97
int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result)
Definition stubmpi.h:293
int MPI_Finalize()
Definition stubmpi.h:182
int MPI_Query_thread(int *provided)
Definition stubmpi.h:184
int MPI_Comm_create(MPI_Comm, MPI_Group, MPI_Comm *newcomm)
Definition stubmpi.h:277
#define MPI_REQUEST_NULL
Definition stubmpi.h:55
int MPI_Finalized(int *flag)
Definition stubmpi.h:183
#define MPI_STATUS_IGNORE
Definition stubmpi.h:21
#define MPI_BYTE
Definition stubmpi.h:78
int MPI_Recv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Status *)
Definition stubmpi.h:222
#define MPI_STATUSES_IGNORE
Definition stubmpi.h:22
int MPI_Comm
Definition stubmpi.h:24
#define MPI_IDENT
Definition stubmpi.h:45
int MPI_Issend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition stubmpi.h:217
int MPI_Info_create(MPI_Info *)
Definition stubmpi.h:337
int MPI_Init_thread(int *, char ***, int, int *provided)
Definition stubmpi.h:180
int MPI_Info_free(MPI_Info *)
Definition stubmpi.h:338
unsigned int MPI_Comm_size(MPI_Comm, int *size)
Definition stubmpi.h:213
int MPI_Buffer_attach(void *, int)
Definition stubmpi.h:187
int MPI_Abort(MPI_Comm, int code)
Definition stubmpi.h:273
int MPI_Comm_split(MPI_Comm, int, int, MPI_Comm *newcomm)
Definition stubmpi.h:262
int MPI_Group_translate_ranks(MPI_Group, int, const int[], MPI_Group, int ranks2[])
Definition stubmpi.h:162
int MPI_Comm_rank(MPI_Comm, int *rank)
Definition stubmpi.h:212
int MPI_Comm_set_errhandler(MPI_Comm, MPI_Errhandler)
Definition stubmpi.h:325
int MPI_Comm_free(MPI_Comm *comm)
Definition stubmpi.h:288
double MPI_Wtime()
Definition stubmpi.h:327
int MPI_Comm_get_attr(MPI_Comm, int, void *, int *)
Definition stubmpi.h:260
#define MPI_COMM_TYPE_SHARED
Definition stubmpi.h:64
int MPI_Info
Definition stubmpi.h:30
int MPI_Type_commit(MPI_Datatype *)
Definition stubmpi.h:96
int MPI_Group_free(MPI_Group *)
Definition stubmpi.h:174
int MPI_Allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, MPI_Comm)
Definition stubmpi.h:255
int MPI_Bsend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition stubmpi.h:220
int MPI_Irecv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition stubmpi.h:221
#define MPI_ERR_COUNT
Definition stubmpi.h:37
int MPI_Comm_group(MPI_Comm, MPI_Group *group)
Definition stubmpi.h:283
int MPI_Barrier(MPI_Comm)
Definition stubmpi.h:275
int MPI_Group_incl(MPI_Group, int, const int[], MPI_Group *)
Definition stubmpi.h:169
int MPI_Testsome(int, MPI_Request *, int *outcount, int *, MPI_Status *)
Definition stubmpi.h:201
int MPI_Get_count(MPI_Status *, MPI_Datatype, int *count)
Definition stubmpi.h:206
int MPI_Type_create_struct(int, const int[], const MPI_Aint[], const MPI_Datatype[], MPI_Datatype *)
Definition stubmpi.h:95
struct MPI_Status MPI_Status
int MPI_Initialized(int *flag)
Definition stubmpi.h:181
int MPI_Init(int *, char ***)
Definition stubmpi.h:179
#define MPI_MAX_ERROR_STRING
Definition stubmpi.h:42
int MPI_Test(MPI_Request *, int *flag, MPI_Status *)
Definition stubmpi.h:190
int MPI_Op_free(MPI_Op *)
Definition stubmpi.h:333
int MPI_Comm_split_type(MPI_Comm comm, int split_type, int, MPI_Info, MPI_Comm *newcomm)
Definition stubmpi.h:268
int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, int, MPI_Comm)
Definition stubmpi.h:251
#define MPI_UNDEFINED
Definition stubmpi.h:26
int MPI_Datatype
Definition stubmpi.h:74
int MPI_Testany(int, MPI_Request[], int *index, int *flag, MPI_Status *)
Definition stubmpi.h:195
int me
Definition test_binsorter.cc:10
static constexpr ProcessID nproc
Rank bound for the hand-written maps, which use the ranks 0, 1 and 2.
Definition test_lbdeux.cc:24
void e()
Definition test_sig.cc:75
double source(const coordT &r)
Definition testperiodic.cc:48
const char * status[2]
Definition testperiodic.cc:43
Implements Mutex, MutexFair, Spinlock, ConditionVariable.