MADNESS  0.10.1
safempi.h
Go to the documentation of this file.
1 /*
2  This file is part of MADNESS.
3 
4  Copyright (C) 2007,2010 Oak Ridge National Laboratory
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10 
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  GNU General Public License for more details.
15 
16  You should have received a copy of the GNU General Public License
17  along with this program; if not, write to the Free Software
18  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 
20  For more information please contact:
21 
22  Robert J. Harrison
23  Oak Ridge National Laboratory
24  One Bethel Valley Road
25  P.O. Box 2008, MS-6367
26 
27  email: harrisonrj@ornl.gov
28  tel: 865-241-3937
29  fax: 865-572-0680
30 
31  $Id$
32 */
33 #ifndef MADNESS_WORLD_SAFEMPI_H__INCLUDED
34 #define MADNESS_WORLD_SAFEMPI_H__INCLUDED
35 
36 /// \file safempi.h
37 /// \brief Serializes calls to MPI in case it does not support THREAD_MULTIPLE
38 
39 #include <madness/madness_config.h>
40 
41 #ifdef STUBOUTMPI
42 #include <madness/world/stubmpi.h>
43 #else
44 
45 //#ifdef SEEK_SET
46 //#undef SEEK_SET
47 //#endif
48 //#ifdef SEEK_CUR
49 //#undef SEEK_CUR
50 //#endif
51 //#ifdef SEEK_END
52 //#undef SEEK_END
53 //#endif
54 
55 #ifdef MADNESS_MPI_HEADER
56 # include MADNESS_MPI_HEADER
57 #else
58 # include <mpi.h>
59 #endif
60 
61 #endif
62 
63 
64 #if MADNESS_MPI_THREAD_LEVEL == MPI_THREAD_SERIALIZED
65 # define MADNESS_SERIALIZES_MPI
66 #endif
67 
68 
69 
72 #include <iostream>
73 #include <csignal>
74 #include <cstdlib>
75 #include <cstring>
76 #include <memory>
77 #include <sstream>
78 
79 #define MADNESS_MPI_TEST(condition) \
80  { \
81  int mpi_error_code = condition; \
82  if(mpi_error_code != MPI_SUCCESS) throw ::SafeMPI::Exception(mpi_error_code); \
83  }
84 
85 namespace SafeMPI {
86 
87  extern madness::SCALABLE_MUTEX_TYPE charon; // Inside safempi.cc
88 #ifdef MADNESS_SERIALIZES_MPI
89 #define SAFE_MPI_GLOBAL_MUTEX madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
90 #else
91 #define SAFE_MPI_GLOBAL_MUTEX
92 #endif
93 
94  /// tags in [1,999] ... allocated once by unique_reserved_tag
95  ///
96  /// tags in [1000,1023] ... statically assigned here
97  ///
98  /// tags in [1024,4095] ... allocated round-robin by unique_tag
99  ///
100  /// tags in [4096,8191] ... reserved for huge msg exchange by RMI
101  ///
102  /// tags in [8192,MPI::TAG_UB] ... not used/managed by madness
103 
104  static const int RMI_TAG = 1023;
105  static const int MPIAR_TAG = 1001;
106  static const int DEFAULT_SEND_RECV_TAG = 1000;
107 
108  // Forward declarations
109  class Intracomm;
110  extern Intracomm COMM_WORLD;
111  inline int Finalize();
112 
113  /// Check MPI initialization status
114 
115  /// \return \c true if MPI has been initialized, \c false otherwise.
116  inline bool Is_initialized() {
117  int initialized = 0;
119  return (initialized != 0);
120  }
121 
122  /// Check MPI finalization status
123 
124  /// \return \c true if MPI has been finalized, \c false otherwise.
125  inline bool Is_finalized() {
126  int flag = 0;
127  MPI_Finalized(&flag);
128  return flag != 0;
129  }
130 
131  namespace detail {
132  /// Initialize SafeMPI::COMM_WORLD
133  inline void init_comm_world();
134 
135  inline void print_mpi_error(const int rc, const char* function,
136  const int line, const char* file)
137  {
138  int len = 0;
139  char error_string[MPI_MAX_ERROR_STRING];
140  MPI_Error_string(rc, error_string, &len);
141  std::cerr << "!!! MPI ERROR (" << rc << ") in " << function <<
142  " at " << file << "(" << line << "): " << error_string << "\n";
143  }
144 
145  } // namespace detail
146 
147  /// SafeMPI exception object
148 
149  /// This exception is thrown whenever an MPI error occurs.
150  class Exception : public std::exception {
151  private:
154  public:
155 
156  Exception(const int mpi_error) throw() {
157  int len = 0;
158  if(MPI_Error_string(mpi_error, mpi_error_string_, &len) != MPI_SUCCESS)
159  std::strncpy(mpi_error_string_, "UNKNOWN MPI ERROR!", MPI_MAX_ERROR_STRING);
160  }
161 
162  Exception(const int mpi_error, const int nstatuses,
163  const int* indices,
164  MPI_Status* const statuses) noexcept {
165  try {
166  if (mpi_error == MPI_ERR_IN_STATUS) {
167  std::ostringstream oss;
168  for(auto s=0; s!=nstatuses; ++s) {
169  int len = 0;
170  auto status_error = statuses[s].MPI_ERROR;
171  if (status_error != MPI_SUCCESS) {
172  oss << "request " << indices[s] << ":";
173  if (MPI_Error_string(status_error, mpi_error_string_, &len) != MPI_SUCCESS)
174  oss << " unknown error!" << std::endl;
175  else
176  oss << mpi_error_string_ << std::endl;
177  }
178  }
179  mpi_statuses_error_string_ = oss.str();
180  }
181  }
182  catch (...) {}
183 
184  int len = 0;
185  if(MPI_Error_string(mpi_error, mpi_error_string_, &len) != MPI_SUCCESS)
186  std::strncpy(mpi_error_string_, "UNKNOWN MPI ERROR!", MPI_MAX_ERROR_STRING);
187  }
188 
189  Exception(const Exception& other) throw() {
190  std::strncpy(mpi_error_string_, other.mpi_error_string_, MPI_MAX_ERROR_STRING);
191  try {
192  mpi_statuses_error_string_ = other.mpi_statuses_error_string_;
193  } catch(...) { mpi_statuses_error_string_.clear(); }
194  }
195 
196  Exception& operator=(const Exception& other) {
198  try {
200  } catch(...) { mpi_statuses_error_string_.clear(); }
201  return *this;
202  }
203 
204  virtual ~Exception() throw() { }
205 
206  virtual const char* what() const throw() { return mpi_error_string_; }
207  bool can_elaborate() const noexcept {
208  return !mpi_statuses_error_string_.empty();
209  }
210  const char* elaborate() const noexcept {
211  return mpi_statuses_error_string_.c_str();
212  }
213 
214  friend std::ostream& operator<<(std::ostream& os, const Exception& e) {
215  os << e.what();
216  if (e.can_elaborate()) {
217  os << e.elaborate();
218  }
219  return os;
220  }
221  }; // class Exception
222 
223 
224  class Status {
225  private:
227 
228  public:
229  // Constructors
230  Status(void) : status_() { }
231  Status(const Status &other) : status_(other.status_) { }
232  Status(MPI_Status other) : status_(other) { }
233 
234  // Assignment operators
235  Status& operator=(const Status &other) {
236  status_ = other.status_;
237  return *this;
238  }
239 
240  Status& operator=(const MPI_Status other) {
241  status_ = other;
242  return *this;
243  }
244 
245  // C/C++ cast and assignment
246  operator MPI_Status*() { return &status_; }
247 
248  operator MPI_Status() const { return status_; }
249 
250 // bool Is_cancelled() const {
251 // int flag = 0;
252 // MADNESS_MPI_TEST(MPI_Test_cancelled(const_cast<MPI_Status*>(&status_), &flag));
253 // return flag != 0;
254 // }
255 //
256 // int Get_elements(const MPI_Datatype datatype) const {
257 // int elements = 0;
258 // MADNESS_MPI_TEST(MPI_Get_elements(const_cast<MPI_Status*>(&status_), datatype, &elements));
259 // return elements;
260 // }
261 
262  int Get_count(const MPI_Datatype datatype) const {
263  int count = 0;
264  MADNESS_MPI_TEST(MPI_Get_count(const_cast<MPI_Status*>(&status_), datatype, &count));
265  return count;
266  }
267 
268 // void Set_cancelled(bool flag) {
269 // MADNESS_MPI_TEST(MPI_Status_set_cancelled(&status_, flag));
270 // }
271 //
272 // void Set_elements( const MPI_Datatype &v2, int v3 ) {
273 // MADNESS_MPI_TEST(MPI_Status_set_elements(&status_, v2, v3 ));
274 // }
275 
276  int Get_source() const { return status_.MPI_SOURCE; }
277 
278  int Get_tag() const { return status_.MPI_TAG; }
279 
280  int Get_error() const { return status_.MPI_ERROR; }
281 
283 
284  void Set_tag(int tag) { status_.MPI_TAG = tag; }
285 
287  }; // class Status
288 
289  class Request {
290  // Note: This class was previously derived from MPI::Request, but this
291  // was changed with the removal of the MPI C++ bindings. Now this class
292  // only implements the minimum functionality required by MADNESS. Feel
293  // free to add more functionality as needed.
294 
295  private:
297 
298  public:
299 
300  // Constructors
302  Request(MPI_Request other) : request_(other) { }
303  Request(const Request& other) : request_(other.request_) { }
304 
305  // Assignment operators
306  Request& operator=(const Request &other) {
307  request_ = other.request_;
308  return *this;
309  }
310 
311  Request& operator=(const MPI_Request& other) {
312  request_ = other;
313  return *this;
314  }
315 
316  // logical
317  bool operator==(const Request &other) { return (request_ == other.request_); }
318  bool operator!=(const Request &other) { return (request_ != other.request_); }
319 
320  // C/C++ cast and assignment
321  operator MPI_Request*() { return &request_; }
322  operator MPI_Request() const { return request_; }
323 
324  static bool Testany(int count, Request* requests, int& index, Status& status) {
325  MADNESS_ASSERT(requests != nullptr);
326  int flag;
327  std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[count]);
328 
329  // Copy requests to an array that can be used by MPI
330  for(int i = 0; i < count; ++i)
331  mpi_requests[i] = requests[i].request_;
332  {
334  MADNESS_MPI_TEST(MPI_Testany(count, mpi_requests.get(), &index, &flag, status));
335  }
336  // Copy results from MPI back to the original array
337  for(int i = 0; i < count; ++i)
338  requests[i].request_ = mpi_requests[i];
339  return flag != 0;
340  }
341 
342  static bool Testany(int count, Request* requests, int& index) {
343  MADNESS_ASSERT(requests != nullptr);
344  int flag;
345  std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[count]);
346 
347  // Copy requests to an array that can be used by MPI
348  for(int i = 0; i < count; ++i)
349  mpi_requests[i] = requests[i].request_;
350  {
352  MADNESS_MPI_TEST(MPI_Testany(count, mpi_requests.get(), &index, &flag, MPI_STATUS_IGNORE));
353  }
354  // Copy results from MPI back to the original array
355  for(int i = 0; i < count; ++i)
356  requests[i] = mpi_requests[i];
357  return flag != 0;
358  }
359 
360  static int Testsome(int incount, Request* requests, int* indices, Status* statuses) {
361  MADNESS_ASSERT(requests != nullptr);
362  MADNESS_ASSERT(indices != nullptr);
363  MADNESS_ASSERT(statuses != nullptr);
364 
365  int outcount = 0;
366  std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[incount]);
367  std::unique_ptr<MPI_Status[]> mpi_statuses(new MPI_Status[incount]);
368  for(int i = 0; i < incount; ++i)
369  mpi_requests[i] = requests[i].request_;
370  {
372  { // print out the status vars for the failed requests
373  auto mpi_error_code =
374  MPI_Testsome(incount, mpi_requests.get(), &outcount,
375  indices, mpi_statuses.get());
376  if (mpi_error_code != MPI_SUCCESS) {
377  throw ::SafeMPI::Exception(mpi_error_code, outcount, indices, mpi_statuses.get());
378  }
379  }
380  }
381  for(int i = 0; i < incount; ++i) {
382  requests[i] = mpi_requests[i];
383  statuses[i] = mpi_statuses[i];
384  }
385  return outcount;
386  }
387 
388  static int Testsome(int incount, Request* requests, int* indices) {
389  int outcount = 0;
390  std::unique_ptr<MPI_Request[]> mpi_requests(new MPI_Request[incount]);
391  for(int i = 0; i < incount; ++i)
392  mpi_requests[i] = requests[i].request_;
393  {
395  MADNESS_MPI_TEST( MPI_Testsome( incount, mpi_requests.get(), &outcount, indices, MPI_STATUSES_IGNORE));
396  }
397  for(int i = 0; i < incount; ++i)
398  requests[i] = mpi_requests[i];
399  return outcount;
400  }
401 
402 
404  int flag;
406  return flag != 0;
407  }
408 
412  }
413 
415  int flag;
417  return flag != 0;
418  }
419 
420  bool Test() {
422  return Test_got_lock_already();
423  }
424  }; // class Request
425 
426  /// Wrapper around MPI_Group. Has a shallow copy constructor. Usually deep copy is not needed, but can be created
427  /// via Group::Incl().
428  class Group {
429  public:
430  Group Incl(int n, const int* ranks) const {
431  // MPI <3 interface lacks explicit const sanitation
432  Group result(std::shared_ptr<Impl>(new Impl(*pimpl, n, const_cast<int*>(ranks))));
433  return result;
434  }
435 
436  void Translate_ranks(int nproc, const int* ranks1, const Group& grp2, int* ranks2) const {
437  // MPI <3 interface lacks explicit const sanitation
440  const_cast<int*>(ranks1), grp2.pimpl->group, ranks2));
441  }
442 
443  MPI_Group group() const {
445  return pimpl->group;
446  }
447 
448  Group(const Group& other) : pimpl(other.pimpl) { }
449 
450  private:
451 
452  struct Impl {
454 
455  Impl(MPI_Comm comm) {
457  }
458 
459  Impl(const Impl& other, int n, const int* ranks) {
460  // MPI <3 interface lacks explicit const sanitation
462  const_cast<int*>(ranks), & group));
463  }
464 
465  ~Impl() {
466  if(Is_initialized()) {
467  const int mpi_error_code = MPI_Group_free(&group);
468  if(mpi_error_code != MPI_SUCCESS)
469  ::SafeMPI::detail::print_mpi_error(mpi_error_code,
470  "SafeMPI::Group::Impl::~Impl()", __LINE__, __FILE__);
471  }
472  }
473 
474  }; // struct Impl
475 
476  friend class Intracomm;
477 
478  Group() : pimpl() { }
479 
480  // only Intracomm will use this
481  Group(MPI_Comm comm) : pimpl(new Impl(comm)) { }
482 
483  // only myself will use this
484  Group(const std::shared_ptr<Impl>& p) : pimpl(p) { }
485 
486  std::shared_ptr<Impl> pimpl;
487  }; // class Group
488 
489  /// Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy
490  class Intracomm {
491 
492  static bool Comm_compare(const MPI_Comm& comm1, const MPI_Comm& comm2) {
493  int compare_result;
494  const int result = MPI_Comm_compare(comm1, comm2, &compare_result);
495  return ((result == MPI_SUCCESS) && (compare_result == MPI_IDENT));
496  }
497 
498  struct Impl {
500  int me;
501  int numproc;
502  bool owner;
503 
504  int utag; // Only used by main thread so no volatile or mutex needed
505  int urtag;// Ditto
506 
507  Impl(const MPI_Comm& c, int m, int n, bool o) :
508  comm(c), me(m), numproc(n), owner(o), utag(1024), urtag(1)
510 
511  ~Impl() {
514  }
515  }
516 
517  /// Returns a unique tag for temporary use (1023<tag<=4095)
518 
519  /// These tags are intended for one time use to avoid tag
520  /// collisions with other messages around the same time period.
521  /// It simply increments/wraps a counter and returns the next
522  /// legal value.
523  ///
524  /// So that send and receiver agree on the tag all processes
525  /// need to call this routine in the same sequence.
526  int unique_tag() {
527  // RJH removed mutex since ordering requirement across processes means
528  // there can never be any thread contention.
529  // Cannot use MPI mutex for anything else!
530  // It will preprocess to nothing for MPI_THREAD_MULTIPLE!
531  //madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
532  int result = utag++;
533  if (utag >= 4095) utag = 1024;
534  return result;
535  }
536 
537  /// \return the period of repeat of unique tags produces by unique_tag()
538  static int unique_tag_period() {
539  const auto min_tag_value = 1024;
540  const auto max_tag_value = 4094;
541  return max_tag_value - min_tag_value + 1;
542  }
543 
544  /// Returns a unique tag reserved for long-term use (0<tag<1000)
545 
546  /// Get a tag from this routine for long-term/repeated use.
547  ///
548  /// Tags in [1000,1023] are statically assigned.
550  // RJH removed mutex since ordering requirement across processes means
551  // Cannot use MPI mutex for anything else!
552  // It will preprocess to nothing for MPI_THREAD_MULTIPLE!
553  // madness::ScopedMutex<madness::SCALABLE_MUTEX_TYPE> obolus(SafeMPI::charon);
554  int result = urtag++;
555  if (result >= 1000) MADNESS_EXCEPTION( "too many reserved tags in use" , result );
556  return result;
557  }
558 
559  };
560  std::shared_ptr<Impl> pimpl;
561 
563  friend int Finalize();
564 
565  // For internal use only. Do not try to call this constructor. It is
566  // only used to construct Intarcomm in Create().
567  Intracomm(const std::shared_ptr<Impl>& i) : pimpl(i) { }
568 
569  // Not allowed
570  Intracomm& operator=(const Intracomm& other);
571 
572  // makes an uninitialized ptr
573  Intracomm() : pimpl(nullptr) {}
574 
575  public:
576  struct WorldInitObject;
577 
578  // For internal use only. Do not try to call this constructor. It is
579  // only used to construct COMM_WORLD.
580  Intracomm(const WorldInitObject&);
581 
582  explicit Intracomm(const MPI_Comm& comm, bool take_ownership_of_comm = true) :
583  pimpl()
584  {
586  int rank = -1, size = -1;
587  MADNESS_MPI_TEST(MPI_Comm_rank(comm, &rank));
588  MADNESS_MPI_TEST(MPI_Comm_size(comm, &size));
589  take_ownership_of_comm =
590  take_ownership_of_comm && (! Comm_compare(comm, MPI_COMM_WORLD));
591  pimpl.reset(new Impl(comm, rank, size, take_ownership_of_comm));
592  }
593 
594  Intracomm(const Intracomm& other) : pimpl(other.pimpl) { }
595 
597 
598  /**
599  * This collective operation creates a new \c Intracomm from an
600  * \c Intracomm::Group object. Must be called by all processes that
601  * belong to this communicator, but not all must use the same \c group .
602  * Thus this \c Intracomm can be partitioned into several \c Intracomm
603  * objects with one call.
604  *
605  * @param group Intracomm::Group describing the Intracomm object to be
606  * created (\c Intracomm::Get_group() and \c Intracomm::Group::Incl() )
607  * @return a new Intracomm object
608  */
609  Intracomm Create(Group group) const {
612  MPI_Comm group_comm;
613  MADNESS_MPI_TEST(MPI_Comm_create(pimpl->comm, group.group(), &group_comm));
614  int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me));
615  int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc));
616  return Intracomm(std::shared_ptr<Impl>(new Impl(group_comm, me, nproc, true)));
617  }
618 
619  static const int UNDEFINED_COLOR = MPI_UNDEFINED;
620  /**
621  * This collective operation creates a new \c Intracomm using
622  * the MPI_Comm_split. Must be called by all processes that
623  * belong to this communicator. Each caller must provide Color of the new Intracomm
624  * and Key (this controls the rank within the new Intracomm;
625  * ties are broken by the rank in this Intracomm).
626  *
627  * @param Color Specifies the new Intracomm that the calling process is to be assigned to.
628  * The value of color must be non-negative. If Color=UNDEFINED_COLOR then
629  * an uninitialized Intracomm object will be produced.
630  * @param Key The relative rank of the calling process in the group of the new Intracomm.
631  * If omitted, each communicator's ranks will be determined by
632  * the rank in the host communicator.
633  * @return a new Intracomm object
634  */
635  Intracomm Split(int Color, int Key = 0) const {
638  MPI_Comm group_comm;
639  MADNESS_MPI_TEST(MPI_Comm_split(pimpl->comm, Color, Key, &group_comm));
640  if (group_comm != MPI_COMM_NULL) {
641  int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me));
642  int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc));
643  return Intracomm(std::shared_ptr<Impl>(new Impl(group_comm, me, nproc, true)));
644  }
645  else
646  return Intracomm();
647  }
648 
651  /**
652  * This collective operation creates a new \c Intracomm using
653  * the MPI_Comm_split_type. Must be called by all processes that
654  * belong to this communicator. Each caller must provide the split type
655  * and Key (this controls the rank within the new Intracomm;
656  * ties are broken by the rank in this Intracomm).
657  *
658  * @param Type Controls how this Intracomm will be split.
659  * The value can only be UNDEFINED_SPLIT_TYPE or SHARED_SPLIT_TYPE.
660  * If Type=UNDEFINED_SPLIT_TYPE then
661  * an uninitialized Intracomm object will be produced.
662  * @param Key The relative rank of the calling process in the group of the new Intracomm.
663  * If omitted, each communicator's ranks will be determined by
664  * the rank in the host communicator.
665  * @return a new Intracomm object
666  */
667  Intracomm Split_type(int Type, int Key = 0) const {
670  MPI_Comm group_comm;
671  MPI_Info info;
672  MPI_Info_create(&info);
673  MADNESS_MPI_TEST(MPI_Comm_split_type(pimpl->comm, Type, Key, info, &group_comm));
674  MPI_Info_free(&info);
675  if (group_comm != MPI_COMM_NULL) {
676  int me; MADNESS_MPI_TEST(MPI_Comm_rank(group_comm, &me));
677  int nproc; MADNESS_MPI_TEST(MPI_Comm_size(group_comm, &nproc));
678  return Intracomm(std::shared_ptr<Impl>(new Impl(group_comm, me, nproc, true)));
679  }
680  else
681  return Intracomm();
682  }
683 
684  /**
685  * Clones this Intracomm object
686  *
687  * @return a (deep) copy of this Intracomm object
688  */
689  Intracomm Clone() const {
690  return Create(this->Get_group());
691  }
692 
693  bool operator==(const Intracomm& other) const {
694  return (pimpl == other.pimpl) || ((pimpl && other.pimpl) &&
695  Comm_compare(pimpl->comm, other.pimpl->comm));
696  }
697 
698  /**
699  * This local operation returns the Intracomm::Group object corresponding to this intracommunicator
700  * @return the Intracomm::Group object corresponding to this intracommunicator
701  */
702  Group Get_group() const {
705  Group group(pimpl->comm);
706  return group;
707  }
708 
711  return pimpl->comm;
712  }
713 
714  int Get_rank() const {
716  return pimpl->me;
717  }
718 
719  int Get_size() const {
721  return pimpl->numproc;
722  }
723 
724  Request Isend(const void* buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const {
727  Request request;
728  MADNESS_MPI_TEST(MPI_Isend(const_cast<void*>(buf), count, datatype, dest,tag, pimpl->comm, request));
729  return request;
730  }
731 
732  Request Issend(const void* buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const {
735  Request request;
736  MADNESS_MPI_TEST(MPI_Issend(const_cast<void*>(buf), count, datatype, dest,tag, pimpl->comm, request));
737  return request;
738  }
739 
740  Request Irecv(void* buf, const int count, const MPI_Datatype datatype, const int src, const int tag) const {
743  Request request;
744  MADNESS_MPI_TEST(MPI_Irecv(buf, count, datatype, src, tag, pimpl->comm, request));
745  return request;
746  }
747 
748  void Send(const void* buf, const int count, const MPI_Datatype datatype, int dest, int tag) const {
751  MADNESS_MPI_TEST(MPI_Ssend(const_cast<void*>(buf), count, datatype, dest, tag, pimpl->comm));
752  }
753 
754 #ifdef MADNESS_USE_BSEND_ACKS
755  void Bsend(const void* buf, size_t count, const MPI_Datatype datatype, int dest, int tag) const {
758  if (count>10 || datatype!=MPI_BYTE) MADNESS_EXCEPTION("Bsend: this protocol is only for 1-byte acks", count );
759  MADNESS_MPI_TEST(MPI_Bsend(const_cast<void*>(buf), count, datatype, dest, tag, pimpl->comm));
760  }
761 #endif // MADNESS_USE_BSEND_ACKS
762 
763  void Recv(void* buf, const int count, const MPI_Datatype datatype, const int source, const int tag, MPI_Status& status) const {
766  MADNESS_MPI_TEST(MPI_Recv(buf, count, datatype, source, tag, pimpl->comm, &status));
767  }
768 
769  void Recv(void* buf, const int count, const MPI_Datatype datatype, const int source, const int tag) const {
772  MADNESS_MPI_TEST(MPI_Recv(buf, count, datatype, source, tag, pimpl->comm, MPI_STATUS_IGNORE));
773  }
774 
775  void Bcast(void* buf, size_t count, const MPI_Datatype datatype, const int root) const {
778  MADNESS_MPI_TEST(MPI_Bcast(buf, count, datatype, root, pimpl->comm));
779  }
780 
781  void Reduce(const void* sendbuf, void* recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op, const int root) const {
784  MADNESS_MPI_TEST(MPI_Reduce(const_cast<void*>(sendbuf), recvbuf, count, datatype, op, root, pimpl->comm));
785  }
786 
787  void Allreduce(const void* sendbuf, void* recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const {
790  MADNESS_MPI_TEST(MPI_Allreduce(const_cast<void*>(sendbuf), recvbuf, count, datatype, op, pimpl->comm));
791  }
792  bool Get_attr(int key, void* value) const {
794  int flag = 0;
796  MADNESS_MPI_TEST(MPI_Comm_get_attr(pimpl->comm, key, value, &flag));
797  return flag != 0;
798  }
799 
800  void Abort(int code=1) const {
801  // if SIGABRT has a handler, call std::abort to allow it be caught,
802  // else call MPI_Abort which does not seem to call abort at all,
803  // instead sends SIGTERM followed by SIGKILL
804  // if have a custom signal handler for SIGABRT (i.e. we are running under a
805  // debugger) then call abort()
806  struct sigaction sa;
807  auto rc = sigaction(SIGABRT, NULL, &sa);
808  if (rc == 0 && sa.sa_handler != SIG_DFL) {
809  std::abort();
810  } else {
811  MPI_Abort(pimpl->comm, code);
812  }
813  }
814 
815  void Barrier() const {
819  }
820 
821  /// Returns a unique tag for temporary use (1023<tag<4095)
822 
823  /// These tags are intended for one time use to avoid tag
824  /// collisions with other messages around the same time period.
825  /// It simply increments/wraps a counter and returns the next
826  /// legal value.
827  ///
828  /// So that send and receiver agree on the tag all processes
829  /// need to call this routine in the same sequence.
830  int unique_tag() {
832  return pimpl->unique_tag();
833  }
834 
835  /// \return the period of repeat of unique tags produces by unique_tag()
836  static int unique_tag_period() {
837  return Impl::unique_tag_period();
838  }
839 
840  /// Returns a unique tag reserved for long-term use (0<tag<1000)
841 
842  /// Get a tag from this routine for long-term/repeated use.
843  ///
844  /// Tags in [1000,1023] are statically assigned.
847  return pimpl->unique_reserved_tag();
848  }
849 
850  /// Construct info about a binary tree with given root
851 
852  /// Constructs a binary tree spanning the communicator with
853  /// process root as the root of the tree. Returns the logical
854  /// parent and children in the tree of the calling process. If
855  /// there is no parent/child the value -1 will be set.
856  void binary_tree_info(int root, int& parent, int& child0, int& child1);
857 
858  }; // class Intracomm
859 
860  namespace detail {
861 
862  /// Initialize SafeMPI::COMM_WORLD
863  inline void init_comm_world() {
867  }
868 
869  } // namespace detail
870 
871 
872  /// Analogous to MPI_Init_thread
873 
874  /// \param argc the number of arguments in argv
875  /// \param argv the vector of command-line arguments
876  /// \param requested the desired thread level
877  /// \return provided thread level
878  inline int Init_thread(int & argc, char **& argv, int requested) {
879  int provided = 0;
880  MADNESS_MPI_TEST(MPI_Init_thread(&argc, &argv, requested, &provided));
882  return provided;
883  }
884 
885  /// Analogous to MPI_Init_thread
886 
887  /// \param requested the desired thread level
888  /// \return provided thread level
889  inline int Init_thread(int requested) {
890  int argc = 0;
891  char** argv = nullptr;
892  return SafeMPI::Init_thread(argc, argv, requested);
893  }
894 
895  /// Analogous to MPI_Init
896 
897  /// \param argc The number of arguments in argv
898  /// \param argv The vector of command-line arguments
899  inline void Init(int &argc, char **&argv) {
900  MADNESS_MPI_TEST(MPI_Init(&argc, &argv));
902  }
903 
904  /// Analogous to MPI_Init
905  inline void Init() {
906  int argc = 0;
907  char** argv = nullptr;
908  SafeMPI::Init(argc,argv);
909  }
910 
911  /// Analogous to MPI_Finalize
912 
913  /// This returns status rather than throw an
914  /// exception upon failure because this is a "destructor", and throwing from
915  /// destructors is evil.
916  /// \return 0 if successful, nonzero otherwise (see MPI_Finalize() for the
917  /// return codes).
918  inline int Finalize() {
919  SafeMPI::COMM_WORLD.pimpl.reset();
920  const int result = MPI_Finalize();
921  return result;
922  }
923 
924  /// Analogous to MPI_Query_thread
925 
926  /// \return the MPI thread level provided by SafeMPI::Init_thread()
927  inline int Query_thread() {
928  int provided;
930  return provided;
931  }
932 
933  /// Wall time
934 
935  /// \return The current wall time
936  inline double Wtime() { return MPI_Wtime(); }
937 
938  /// Set buffer for \c Bsend .
939 
940  /// \param buffer The buffer to be used by Bsend
941  /// \param size The size of the buffer in Bytes
942  inline void Attach_buffer(void* buffer, int size) {
943  MADNESS_MPI_TEST(MPI_Buffer_attach(buffer, size));
944  }
945 
946  /// Unset the \c Bsend buffer.
947 
948  /// \param[out] buffer The buffer that was used by Bsend
949  inline int Detach_buffer(void *&buffer) {
950  int size = 0;
951  MPI_Buffer_detach(&buffer, &size);
952  return size;
953  }
954 
955  /// Analogous to MPI_Op_create
956  inline MPI_Op Op_create(MPI_User_function *user_fn, int commute) {
957  MPI_Op result;
958  MADNESS_MPI_TEST(MPI_Op_create(user_fn, commute, &result));
959  return result;
960  }
961 
962  /// Analogous to MPI_Op_free
963  inline void Op_free(MPI_Op op) {
965  }
966 
967 } // namespace SafeMPI
968 
969 #endif // MADNESS_WORLD_SAFEMPI_H__INCLUDED
SafeMPI exception object.
Definition: safempi.h:150
std::string mpi_statuses_error_string_
Definition: safempi.h:153
bool can_elaborate() const noexcept
Definition: safempi.h:207
Exception(const int mpi_error)
Definition: safempi.h:156
char mpi_error_string_[MPI_MAX_ERROR_STRING]
Definition: safempi.h:152
Exception & operator=(const Exception &other)
Definition: safempi.h:196
Exception(const int mpi_error, const int nstatuses, const int *indices, MPI_Status *const statuses) noexcept
Definition: safempi.h:162
friend std::ostream & operator<<(std::ostream &os, const Exception &e)
Definition: safempi.h:214
virtual ~Exception()
Definition: safempi.h:204
virtual const char * what() const
Definition: safempi.h:206
const char * elaborate() const noexcept
Definition: safempi.h:210
Exception(const Exception &other)
Definition: safempi.h:189
Definition: safempi.h:428
Group(const std::shared_ptr< Impl > &p)
Definition: safempi.h:484
Group Incl(int n, const int *ranks) const
Definition: safempi.h:430
MPI_Group group() const
Definition: safempi.h:443
std::shared_ptr< Impl > pimpl
Definition: safempi.h:486
Group(MPI_Comm comm)
Definition: safempi.h:481
void Translate_ranks(int nproc, const int *ranks1, const Group &grp2, int *ranks2) const
Definition: safempi.h:436
Group(const Group &other)
Definition: safempi.h:448
Group()
Definition: safempi.h:478
Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy.
Definition: safempi.h:490
Intracomm(const std::shared_ptr< Impl > &i)
Definition: safempi.h:567
Intracomm Clone() const
Definition: safempi.h:689
void Allreduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op) const
Definition: safempi.h:787
static bool Comm_compare(const MPI_Comm &comm1, const MPI_Comm &comm2)
Definition: safempi.h:492
bool Get_attr(int key, void *value) const
Definition: safempi.h:792
static const int SHARED_SPLIT_TYPE
Definition: safempi.h:650
Intracomm Create(Group group) const
Definition: safempi.h:609
Request Irecv(void *buf, const int count, const MPI_Datatype datatype, const int src, const int tag) const
Definition: safempi.h:740
void Bcast(void *buf, size_t count, const MPI_Datatype datatype, const int root) const
Definition: safempi.h:775
void Reduce(const void *sendbuf, void *recvbuf, const int count, const MPI_Datatype datatype, const MPI_Op op, const int root) const
Definition: safempi.h:781
Request Isend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition: safempi.h:724
friend int Finalize()
Analogous to MPI_Finalize.
Definition: safempi.h:918
void binary_tree_info(int root, int &parent, int &child0, int &child1)
Construct info about a binary tree with given root.
Definition: safempi.cc:39
Intracomm Split_type(int Type, int Key=0) const
Definition: safempi.h:667
int Get_rank() const
Definition: safempi.h:714
static int unique_tag_period()
Definition: safempi.h:836
void Abort(int code=1) const
Definition: safempi.h:800
static const int UNDEFINED_SPLIT_TYPE
Definition: safempi.h:649
Intracomm(const MPI_Comm &comm, bool take_ownership_of_comm=true)
Definition: safempi.h:582
int unique_tag()
Returns a unique tag for temporary use (1023<tag<4095)
Definition: safempi.h:830
bool operator==(const Intracomm &other) const
Definition: safempi.h:693
void Barrier() const
Definition: safempi.h:815
std::shared_ptr< Impl > pimpl
Definition: safempi.h:560
Group Get_group() const
Definition: safempi.h:702
Intracomm & operator=(const Intracomm &other)
void Send(const void *buf, const int count, const MPI_Datatype datatype, int dest, int tag) const
Definition: safempi.h:748
Intracomm Split(int Color, int Key=0) const
Definition: safempi.h:635
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000)
Definition: safempi.h:845
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag) const
Definition: safempi.h:769
MPI_Comm & Get_mpi_comm() const
Definition: safempi.h:709
~Intracomm()
Definition: safempi.h:596
void Recv(void *buf, const int count, const MPI_Datatype datatype, const int source, const int tag, MPI_Status &status) const
Definition: safempi.h:763
Intracomm(const Intracomm &other)
Definition: safempi.h:594
int Get_size() const
Definition: safempi.h:719
Intracomm()
Definition: safempi.h:573
static const int UNDEFINED_COLOR
Definition: safempi.h:619
Request Issend(const void *buf, const int count, const MPI_Datatype datatype, const int dest, const int tag) const
Definition: safempi.h:732
Definition: safempi.h:289
bool operator!=(const Request &other)
Definition: safempi.h:318
Request & operator=(const MPI_Request &other)
Definition: safempi.h:311
MPI_Request request_
Definition: safempi.h:296
bool Test(MPI_Status &status)
Definition: safempi.h:409
bool Test_got_lock_already()
Definition: safempi.h:414
static bool Testany(int count, Request *requests, int &index)
Definition: safempi.h:342
static int Testsome(int incount, Request *requests, int *indices)
Definition: safempi.h:388
Request()
Definition: safempi.h:301
Request(MPI_Request other)
Definition: safempi.h:302
static int Testsome(int incount, Request *requests, int *indices, Status *statuses)
Definition: safempi.h:360
bool Test_got_lock_already(MPI_Status &status)
Definition: safempi.h:403
static bool Testany(int count, Request *requests, int &index, Status &status)
Definition: safempi.h:324
Request & operator=(const Request &other)
Definition: safempi.h:306
bool operator==(const Request &other)
Definition: safempi.h:317
Request(const Request &other)
Definition: safempi.h:303
bool Test()
Definition: safempi.h:420
Definition: safempi.h:224
int Get_count(const MPI_Datatype datatype) const
Definition: safempi.h:262
int Get_error() const
Definition: safempi.h:280
void Set_tag(int tag)
Definition: safempi.h:284
Status & operator=(const Status &other)
Definition: safempi.h:235
Status(const Status &other)
Definition: safempi.h:231
MPI_Status status_
Definition: safempi.h:226
int Get_tag() const
Definition: safempi.h:278
Status(MPI_Status other)
Definition: safempi.h:232
int Get_source() const
Definition: safempi.h:276
Status(void)
Definition: safempi.h:230
void Set_source(int source)
Definition: safempi.h:282
void Set_error(int error)
Definition: safempi.h:286
Status & operator=(const MPI_Status other)
Definition: safempi.h:240
Mutex using pthread mutex operations.
Definition: worldmutex.h:131
char * strncpy()
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition: derivatives.cc:72
const double m
Definition: gfit.cc:199
Tensor< double > op(const Tensor< double > &x)
Definition: kain.cc:508
Macros and tools pertaining to the configuration of MADNESS.
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition: madness_exception.h:119
#define MADNESS_ASSERT(condition)
Assert a condition that should be free of side-effects since in release builds this might be a no-op.
Definition: madness_exception.h:134
void init_comm_world()
Initialize SafeMPI::COMM_WORLD.
Definition: safempi.h:863
void print_mpi_error(const int rc, const char *function, const int line, const char *file)
Definition: safempi.h:135
Definition: safempi.cc:35
Intracomm COMM_WORLD
Definition: safempi.cc:67
static const int MPIAR_TAG
Definition: safempi.h:105
int Init_thread(int &argc, char **&argv, int requested)
Analogous to MPI_Init_thread.
Definition: safempi.h:878
void Init(int &argc, char **&argv)
Analogous to MPI_Init.
Definition: safempi.h:899
void Op_free(MPI_Op op)
Analogous to MPI_Op_free.
Definition: safempi.h:963
void Attach_buffer(void *buffer, int size)
Set buffer for Bsend .
Definition: safempi.h:942
madness::SCALABLE_MUTEX_TYPE charon
Definition: safempi.cc:37
int Query_thread()
Analogous to MPI_Query_thread.
Definition: safempi.h:927
int Finalize()
Analogous to MPI_Finalize.
Definition: safempi.h:918
double Wtime()
Wall time.
Definition: safempi.h:936
bool Is_initialized()
Check MPI initialization status.
Definition: safempi.h:116
static const int DEFAULT_SEND_RECV_TAG
Definition: safempi.h:106
static const int RMI_TAG
Definition: safempi.h:104
int Detach_buffer(void *&buffer)
Unset the Bsend buffer.
Definition: safempi.h:949
bool Is_finalized()
Check MPI finalization status.
Definition: safempi.h:125
MPI_Op Op_create(MPI_User_function *user_fn, int commute)
Analogous to MPI_Op_create.
Definition: safempi.h:956
bool initialized()
Check if the MADNESS runtime has been initialized (and not subsequently finalized).
Definition: world.cc:74
void error(const char *msg, int code)
Definition: oldtest.cc:57
static const double c
Definition: relops.cc:10
#define MADNESS_MPI_TEST(condition)
Definition: safempi.h:79
#define SAFE_MPI_GLOBAL_MUTEX
Definition: safempi.h:89
Definition: test_dc.cc:47
Definition: stubmpi.h:12
int MPI_SOURCE
Definition: stubmpi.h:15
int MPI_TAG
Definition: stubmpi.h:16
int MPI_ERROR
Definition: stubmpi.h:17
Definition: safempi.h:452
Impl(MPI_Comm comm)
Definition: safempi.h:455
MPI_Group group
Definition: safempi.h:453
~Impl()
Definition: safempi.h:465
Impl(const Impl &other, int n, const int *ranks)
Definition: safempi.h:459
Definition: safempi.h:498
int numproc
Definition: safempi.h:501
int urtag
Definition: safempi.h:505
int unique_reserved_tag()
Returns a unique tag reserved for long-term use (0<tag<1000)
Definition: safempi.h:549
bool owner
Definition: safempi.h:502
static int unique_tag_period()
Definition: safempi.h:538
int me
Definition: safempi.h:500
int utag
Definition: safempi.h:504
MPI_Comm comm
Definition: safempi.h:499
int unique_tag()
Returns a unique tag for temporary use (1023<tag<=4095)
Definition: safempi.h:526
Impl(const MPI_Comm &c, int m, int n, bool o)
Definition: safempi.h:507
~Impl()
Definition: safempi.h:511
#define MPI_SUCCESS
Definition: stubmpi.h:33
int MPI_Op_free(MPI_Op *op)
Definition: stubmpi.h:258
int MPI_Ssend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition: stubmpi.h:167
#define MPI_ERRORS_RETURN
Definition: stubmpi.h:37
void() MPI_User_function(void *a, void *b, int *len, MPI_Datatype *)
Definition: stubmpi.h:107
int MPI_Info_create(MPI_Info *info)
Definition: stubmpi.h:262
int MPI_Bcast(void *, int, MPI_Datatype, int, MPI_Comm)
Definition: stubmpi.h:173
#define MPI_COMM_NULL
Definition: stubmpi.h:47
int MPI_Error_string(int errorcode, char *string, int *resultlen)
Definition: stubmpi.h:228
int MPI_Isend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition: stubmpi.h:164
int MPI_Op
Definition: stubmpi.h:91
int MPI_Group_free(MPI_Group *group)
Definition: stubmpi.h:122
int MPI_Request
Definition: stubmpi.h:11
#define MPI_ERR_IN_STATUS
Definition: stubmpi.h:36
int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm)
Definition: stubmpi.h:187
#define MPI_COMM_WORLD
Definition: stubmpi.h:24
int MPI_Group
Definition: stubmpi.h:10
int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result)
Definition: stubmpi.h:218
int MPI_Finalize()
Definition: stubmpi.h:130
int MPI_Info_free(MPI_Info *info)
Definition: stubmpi.h:263
int MPI_Query_thread(int *provided)
Definition: stubmpi.h:132
int MPI_Comm_create(MPI_Comm, MPI_Group, MPI_Comm *newcomm)
Definition: stubmpi.h:202
#define MPI_REQUEST_NULL
Definition: stubmpi.h:51
int MPI_Op_create(MPI_User_function *user_fn, int commute, MPI_Op *op)
Definition: stubmpi.h:254
int MPI_Finalized(int *flag)
Definition: stubmpi.h:131
#define MPI_STATUS_IGNORE
Definition: stubmpi.h:20
#define MPI_BYTE
Definition: stubmpi.h:74
int MPI_Recv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Status *)
Definition: stubmpi.h:170
#define MPI_STATUSES_IGNORE
Definition: stubmpi.h:21
int MPI_Comm
Definition: stubmpi.h:23
#define MPI_IDENT
Definition: stubmpi.h:41
int MPI_Issend(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition: stubmpi.h:165
int MPI_Init_thread(int *, char ***, int, int *provided)
Definition: stubmpi.h:128
unsigned int MPI_Comm_size(MPI_Comm, int *size)
Definition: stubmpi.h:161
int MPI_Buffer_attach(void *, int)
Definition: stubmpi.h:135
int MPI_Comm_set_errhandler(MPI_Comm comm, MPI_Errhandler errhandler)
Definition: stubmpi.h:250
int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key, MPI_Info info, MPI_Comm *newcomm)
Definition: stubmpi.h:193
int MPI_Abort(MPI_Comm, int code)
Definition: stubmpi.h:198
int MPI_Group_translate_ranks(MPI_Group, int, const int[], MPI_Group, int ranks2[])
Definition: stubmpi.h:110
int MPI_Comm_rank(MPI_Comm, int *rank)
Definition: stubmpi.h:160
int MPI_Comm_free(MPI_Comm *comm)
Definition: stubmpi.h:213
double MPI_Wtime()
Definition: stubmpi.h:252
int MPI_Comm_get_attr(MPI_Comm, int, void *, int *)
Definition: stubmpi.h:185
#define MPI_COMM_TYPE_SHARED
Definition: stubmpi.h:60
int MPI_Info
Definition: stubmpi.h:29
int MPI_Allreduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, MPI_Comm)
Definition: stubmpi.h:180
int MPI_Bsend(void *, int, MPI_Datatype, int, int, MPI_Comm)
Definition: stubmpi.h:168
int MPI_Irecv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Request *)
Definition: stubmpi.h:169
int MPI_Comm_group(MPI_Comm, MPI_Group *group)
Definition: stubmpi.h:208
int MPI_Barrier(MPI_Comm)
Definition: stubmpi.h:200
int MPI_Testsome(int, MPI_Request *, int *outcount, int *, MPI_Status *)
Definition: stubmpi.h:149
int MPI_Get_count(MPI_Status *, MPI_Datatype, int *count)
Definition: stubmpi.h:154
struct MPI_Status MPI_Status
int MPI_Initialized(int *flag)
Definition: stubmpi.h:129
int MPI_Init(int *, char ***)
Definition: stubmpi.h:127
#define MPI_MAX_ERROR_STRING
Definition: stubmpi.h:38
int MPI_Test(MPI_Request *, int *flag, MPI_Status *)
Definition: stubmpi.h:138
int MPI_Group_incl(MPI_Group group, int n, const int ranks[], MPI_Group *newgroup)
Definition: stubmpi.h:117
int MPI_Buffer_detach(void *buffer, int *size)
Definition: stubmpi.h:136
int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype, MPI_Op, int, MPI_Comm)
Definition: stubmpi.h:176
#define MPI_UNDEFINED
Definition: stubmpi.h:25
int MPI_Datatype
Definition: stubmpi.h:70
int MPI_Testany(int, MPI_Request[], int *index, int *flag, MPI_Status *)
Definition: stubmpi.h:143
int me
Definition: test_binsorter.cc:10
void e()
Definition: test_sig.cc:75
double source(const coordT &r)
Definition: testperiodic.cc:48
const char * status[2]
Definition: testperiodic.cc:43
Implements Mutex, MutexFair, Spinlock, ConditionVariable.