MADNESS 0.10.1
worldrmi.h
Go to the documentation of this file.
1/*
2 This file is part of MADNESS.
3
4 Copyright (C) 2007,2010 Oak Ridge National Laboratory
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
20 For more information please contact:
21
22 Robert J. Harrison
23 Oak Ridge National Laboratory
24 One Bethel Valley Road
25 P.O. Box 2008, MS-6367
26
27 email: harrisonrj@ornl.gov
28 tel: 865-241-3937
29 fax: 865-572-0680
30*/
31
32#ifndef MADNESS_WORLD_WORLDRMI_H__INCLUDED
33#define MADNESS_WORLD_WORLDRMI_H__INCLUDED
34
39#include <sstream>
40#include <utility>
41#include <list>
42#include <memory>
43#include <tuple>
44#include <pthread.h>
45#include <madness/world/print.h>
46
47/*
48 There is just one server thread and it is the only one
49 messing with the recv buffers, so there is no need for
50 mutex on recv related data.
51
52 Multiple threads (including the server) may send hence
53 we need to be careful about send-related data.
54
55 When MPI is initialized we need to use init_thread with
56 multiple required.
57
58 This RMI service operates only in (a clone of) COMM_WORLD. It easy enough
59 to extend to other communicators but the point is to have
60 only one server thread for all possible uses. You just
61 have to translate rank_in_comm into rank_in_world by
62 getting the groups from both communicators using
63 MPI_Comm_group and then creating a map from ranks in
64 comm to ranks in world using MPI_Group_translate_ranks.
65
66 The class is a singleton ... i.e., there is only one instance of it
67 that is made the first time that you call RMI::instance().
68
69 Handler routines should have this type
70
71 typedef void (*rmi_handlerT)(void* buf, size_t nbyte);
72
73 There are few user accessible routines.
74
75 RMI::Request RMI::isend(const void* buf, size_t nbyte, int dest,
76 rmi_handlerT func, unsigned int attr=0)
77 - to send an asynchronous message
78 - RMI::Request has the same interface as SafeMPI::Request
79 (right now it is a SafeMPI::Request but this is not guaranteed)
80
81 void RMI::begin()
82 - to start the server thread
83
84 void RMI::end()
85 - to terminate the server thread
86
87 bool RMI::get_debug()
88 - to get the debug flag
89
90 void RMI::set_debug(bool)
91 - to set the debug flag
92
93*/
94
95/**
96 \file worldrmi.h
97 \brief Lowest level API for sending active messages --- you should probably be looking at worldam.h instead.
98 \addtogroup parallel_runtime
99 */
100
101namespace madness {
102
103 /// This is the generic low-level interface for a message handler
104 typedef void (*rmi_handlerT)(void* buf, size_t nbyte);
105 typedef std::ptrdiff_t rel_fn_ptr_t;
106
107 struct qmsg {
108 typedef uint16_t counterT; //!< counter for ordered messages
109 typedef uint32_t attrT; //!< attributes of the message; high 16 bits are the counter
110 size_t len;
112 int i; // buffer index
116
118 : len(len), func(func), i(i), src(src), attr(attr), count(count) {}
119
120 // N.B. since msg counters in same batch might wrap around 0, need to sort buckets defined by the 2 highest
121 // bits ... basically we want 11xxx < 00xxx < 01xxx < 10xxx < 11xxx ... assume we only have messages from
122 // at most 2 adjacent buckets, thus only when comparing counters from buckets 00 and 11 reverse the order
123 // P.S. we thus assume we won't have to deal with msg sequences > 2^14 (per rank)
124 friend inline bool operator<(const qmsg& a, const qmsg& b) {
125 const auto a_src = a.src;
126 const auto b_src = b.src;
127 if (a_src == b_src) {
128 const auto a_count_bk = a.count >> 14;
129 const auto b_count_bk = b.count >> 14;
130 if (a_count_bk == 0b00 && b_count_bk == 0b11) {
131 return false;
132 } else if (a_count_bk == 0b11 && b_count_bk == 0b00) {
133 return true;
134 } else {
135 return a.count < b.count;
136 }
137 } else {
138 return a_src < b_src;
139 }
140 }
141
142 qmsg() {}
143 }; // struct qmsg
144
145
146 // Holds message passing statistics
147 struct RMIStats {
148 uint64_t nmsg_sent;
149 uint64_t nbyte_sent;
150 uint64_t nmsg_recv;
151 uint64_t nbyte_recv;
153
156 };
157
158 /// This for RMI server thread to manage lifetime of WorldAM messages that it is sending
159 struct RMISendReq {
160 virtual bool TestAndFree() = 0;
161 virtual ~RMISendReq() {} // ESSENTIAL!!
162 };
163
164 /// This class implements the communications server thread and provides the only send interface
165 class RMI {
168
169 /// @return reference to the boolean variable indicating whether this thread is the server thread
170 static bool& is_server_thread_accessor();
171
172 public:
173
175
176 // Choose header length to hold at least sizeof(header) and
177 // also to ensure good alignment of the user payload.
178 static const size_t ALIGNMENT = 64;
179 static const size_t HEADER_LEN = ALIGNMENT;
180 static const attrT ATTR_UNORDERED=0x0;
181 static const attrT ATTR_ORDERED=0x1;
182
184
185 static void set_this_thread_is_server(bool flag = true) { is_server_thread_accessor() = flag;}
187
188 static std::list< std::unique_ptr<RMISendReq> > send_req; // List of outstanding world active messages sent by the server
189
190 private:
191
192 static void clear_send_req() {
193 //std::cout << "clearing server messages " << pthread_self() << std::endl;
194 stats.max_serv_send_q = std::max(stats.max_serv_send_q,uint64_t(send_req.size()));
195 auto it=send_req.begin();
196 while (it != send_req.end()) {
197 if ((*it)->TestAndFree())
198 it = send_req.erase(it);
199 else
200 ++it;
201 }
202 }
203
205#if HAVE_INTEL_TBB
206 : private madness::Mutex
207#else
208 : public madness::ThreadBase, private madness::Mutex
209#endif // HAVE_INTEL_TBB
210 {
211
212 public:
213
214 struct header {
217 }; // struct header
218
219 /// q of huge messages, each msg = {source,nbytes,tag}
220 std::list< std::tuple<int,size_t,int> > hugeq;
221 /// size of the huge message being received into recv_buf[nrecv_]
222 std::size_t huge_msg_len_ = 0;
223
225 const int nproc; // No. of processes in comm world
226 const ProcessID rank; // Rank of this process
227 std::atomic<bool> finished; // True if finished ... atomic seems preferable to volatile
228 std::unique_ptr<counterT[]> send_counters; // used to be volatile but no need
229 std::unique_ptr<counterT[]> recv_counters;
230 std::size_t max_msg_len_;
231 std::size_t nrecv_;
233 std::size_t maxq_;
234 std::unique_ptr<void*[]> recv_buf; // Will be at least ALIGNMENT aligned ... +1 for huge messages
235 std::unique_ptr<SafeMPI::Request[]> recv_req;
236
237 std::unique_ptr<SafeMPI::Status[]> status;
238 std::unique_ptr<int[]> ind;
239 std::unique_ptr<qmsg[]> q;
241
242 static inline bool is_ordered(attrT attr) { return attr & ATTR_ORDERED; }
243
244 void process_some();
245
247 virtual ~RmiTask();
248
249 static void set_rmi_task_is_running(bool flag = true);
250
251#if HAVE_INTEL_TBB
252 void run() {
253 ::madness::binder.bind();
256
257 while (! finished) process_some();
258
261
262 finished = false; // to ensure that RmiTask::exit() that
263 // triggered the exit proceeds to completion
264 }
265#else
266 void run() {
267 ::madness::binder.bind();
269 try {
270 while (! finished) process_some();
271 finished = false;
272 } catch(...) {
273 delete this;
275 throw;
276 }
278 }
279#endif // HAVE_INTEL_TBB
280
281 void exit() {
282 if (debugging)
283 print_error(rank, ":RMI: sending exit request to server thread\n");
284
285 // Set finished flag
286 finished = true;
287 while(finished)
288 myusleep(1000);
289 }
290
291 static void huge_msg_handler(void *buf, size_t nbytein);
292
293 Request isend(const void* buf, size_t nbyte, ProcessID dest, rmi_handlerT func, attrT attr);
294
296
297 void post_recv_buf(int i);
298
299 private:
300
301 /// thread-safely round-robins through tags in [first_tag, first_tag+period) range
302 /// @returns new tag to be used in messaging
303 int unique_tag() const;
304 /// the period of tags returned by unique_tag()
305 /// @warning this bounds how many huge messages each RmiTask will be able to process
306 static constexpr int unique_tag_period() { return 2048; }
307
308 }; // class RmiTask
309
310
311 static std::unique_ptr<RmiTask> task_ptr; // Pointer to the singleton instance
313 static bool debugging; // True if debugging ... used to be volatile but no need
314
315 static const size_t DEFAULT_MAX_MSG_LEN = 3*512*1024; //!< the default size of recv buffers, in bytes; the actual size can be configured by the user via envvar MAD_BUFFER_SIZE
316 static const int DEFAULT_NRECV = 128; //!< the default # of recv buffers; the actual number can be configured by the user via envvar MAD_RECV_BUFFERS
317
318 // Not allowed
319 RMI(const RMI&);
320 RMI& operator=(const RMI&);
321
322 public:
323
324 /// Returns the size of recv buffers, in bytes
325
326 /// @return The size of recv buffers, in bytes
327 /// @note The default value is given by RMI::DEFAULT_MAX_MSG_LEN, can be overridden at runtime by the user via environment variable MAD_BUFFER_SIZE.
328 /// @warning Cannot be smaller than 1024 bytes.
329 static std::size_t max_msg_len() {
331 return task_ptr->max_msg_len_;
332 }
333 static std::size_t maxq() {
335 return task_ptr->maxq_;
336 }
337
338 /// Returns the number of recv buffers
339
340 /// @return The number of recv buffers
341 /// @note The default value is given by RMI::DEFAULT_NRECV, can be overridden at runtime by the user via environment variable MAD_RECV_BUFFERS
342 /// @warning Cannot be smaller than 32.
343 static std::size_t nrecv() {
345 return task_ptr->nrecv_;
346 }
347
348 /// Send a remote method invocation (again you should probably be looking at worldam.h instead)
349
350 /// @param[in] buf Pointer to the data buffer (do not modify until send is completed)
351 /// @param[in] nbyte Size of the data in bytes
352 /// @param[in] dest Process to receive the message
353 /// @param[in] func The function to handle the message on the remote end
354 /// @param[in] attr Attributes of the message (ATTR_UNORDERED or ATTR_ORDERED)
355 /// @return The status as an RMI::Request that presently is a SafeMPI::Request
356 static Request
357 isend(const void* buf, size_t nbyte, ProcessID dest, rmi_handlerT func, unsigned int attr=ATTR_UNORDERED) {
358 if(!task_ptr) {
360 "!! MADNESS RMI error: Attempting to send a message when the RMI thread is not running\n"
361 "!! MADNESS RMI error: This typically occurs when an active message is sent or a remote task is spawned after calling madness::finalize()\n");
362 MADNESS_EXCEPTION("!! MADNESS error: The RMI thread is not running", (task_ptr != nullptr));
363 }
364 return task_ptr->isend(buf, nbyte, dest, func, attr);
365 }
366
367 /// will complain to std::cerr and throw if ASLR is on by making
368 /// sure that address of this function matches across @p comm
369 /// @param[in] comm the communicator
370 static void assert_aslr_off(const SafeMPI::Intracomm& comm = SafeMPI::COMM_WORLD);
371
372 static void begin(const SafeMPI::Intracomm& comm = SafeMPI::COMM_WORLD);
373
374 static void end() {
375 if(task_ptr) {
376 task_ptr->exit();
377 //exit insures that RMI task is completed, therefore it is OK to delete it
378 task_ptr = nullptr;
379 }
380 }
381
382 static void set_debug(bool status) { debugging = status; }
383
384 static bool get_debug() { return debugging; }
385
386 static const RMIStats& get_stats() { return stats; }
387 }; // class RMI
388
389} // namespace madness
390
391#endif // MADNESS_WORLD_WORLDRMI_H__INCLUDED
Interface templates for the archives (serialization).
Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy.
Definition safempi.h:645
Definition safempi.h:320
Mutex using pthread mutex operations.
Definition worldmutex.h:150
void post_pending_huge_msg()
Definition worldrmi.cc:197
RmiTask(const SafeMPI::Intracomm &comm=SafeMPI::COMM_WORLD)
Definition worldrmi.cc:245
std::size_t huge_msg_len_
size of the huge message being received into recv_buf[nrecv_]
Definition worldrmi.h:222
const ProcessID rank
Definition worldrmi.h:226
std::size_t maxq_
Definition worldrmi.h:233
std::unique_ptr< counterT[]> recv_counters
Definition worldrmi.h:229
std::unique_ptr< void *[]> recv_buf
Definition worldrmi.h:234
std::unique_ptr< SafeMPI::Status[]> status
Definition worldrmi.h:237
std::atomic< bool > finished
Definition worldrmi.h:227
static bool is_ordered(attrT attr)
Definition worldrmi.h:242
int n_in_q
Definition worldrmi.h:240
void run()
Definition worldrmi.h:252
std::unique_ptr< counterT[]> send_counters
Definition worldrmi.h:228
std::unique_ptr< qmsg[]> q
Definition worldrmi.h:239
std::unique_ptr< int[]> ind
Definition worldrmi.h:238
static void huge_msg_handler(void *buf, size_t nbytein)
Definition worldrmi.cc:353
long nssend_
Definition worldrmi.h:232
static constexpr int unique_tag_period()
Definition worldrmi.h:306
void post_recv_buf(int i)
Definition worldrmi.cc:219
int unique_tag() const
Definition worldrmi.cc:511
std::size_t nrecv_
Definition worldrmi.h:231
Request isend(const void *buf, size_t nbyte, ProcessID dest, rmi_handlerT func, attrT attr)
SafeMPI::Intracomm comm
Definition worldrmi.h:224
std::unique_ptr< SafeMPI::Request[]> recv_req
Definition worldrmi.h:235
virtual ~RmiTask()
Definition worldrmi.cc:233
std::list< std::tuple< int, size_t, int > > hugeq
q of huge messages, each msg = {source,nbytes,tag}
Definition worldrmi.h:220
std::size_t max_msg_len_
Definition worldrmi.h:230
const int nproc
Definition worldrmi.h:225
void exit()
Definition worldrmi.h:281
void process_some()
Definition worldrmi.cc:62
static void set_rmi_task_is_running(bool flag=true)
Definition worldrmi.cc:439
static const attrT ATTR_UNORDERED
Definition worldrmi.h:180
static void begin(const SafeMPI::Intracomm &comm=SafeMPI::COMM_WORLD)
Definition worldrmi.cc:402
static int testsome_backoff_us
Definition worldrmi.h:183
RMI(const RMI &)
qmsg::counterT counterT
Definition worldrmi.h:166
static bool & is_server_thread_accessor()
Definition worldrmi.cc:56
static void clear_send_req()
Definition worldrmi.h:192
static std::size_t nrecv()
Returns the number of recv buffers.
Definition worldrmi.h:343
static std::size_t maxq()
Definition worldrmi.h:333
static std::size_t max_msg_len()
Returns the size of recv buffers, in bytes.
Definition worldrmi.h:329
static bool debugging
Definition worldrmi.h:313
static bool get_this_thread_is_server()
Definition worldrmi.h:186
static void set_debug(bool status)
Definition worldrmi.h:382
static const size_t HEADER_LEN
Definition worldrmi.h:179
static const attrT ATTR_ORDERED
Definition worldrmi.h:181
static void assert_aslr_off(const SafeMPI::Intracomm &comm=SafeMPI::COMM_WORLD)
Definition worldrmi.cc:387
static std::list< std::unique_ptr< RMISendReq > > send_req
Definition worldrmi.h:188
static const size_t DEFAULT_MAX_MSG_LEN
the default size of recv buffers, in bytes; the actual size can be configured by the user via envvar ...
Definition worldrmi.h:315
static const int DEFAULT_NRECV
the default # of recv buffers; the actual number can be configured by the user via envvar MAD_RECV_BU...
Definition worldrmi.h:316
static bool get_debug()
Definition worldrmi.h:384
static void end()
Definition worldrmi.h:374
static void set_this_thread_is_server(bool flag=true)
Definition worldrmi.h:185
qmsg::attrT attrT
Definition worldrmi.h:167
static std::unique_ptr< RmiTask > task_ptr
Definition worldrmi.h:311
RMI & operator=(const RMI &)
static const size_t ALIGNMENT
Definition worldrmi.h:178
static const RMIStats & get_stats()
Definition worldrmi.h:386
static RMIStats stats
Definition worldrmi.h:312
SafeMPI::Request Request
Definition worldrmi.h:174
static Request isend(const void *buf, size_t nbyte, ProcessID dest, rmi_handlerT func, unsigned int attr=ATTR_UNORDERED)
Send a remote method invocation (again you should probably be looking at worldam.h instead).
Definition worldrmi.h:357
Simplified thread wrapper to hide pthread complexity.
Definition thread.h:171
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_ASSERT(condition)
Assert a condition that should be free of side-effects since in release builds this might be a no-op.
Definition madness_exception.h:134
Intracomm COMM_WORLD
Definition safempi.cc:67
Namespace for all elements and tools of MADNESS.
Definition DFConvergence.h:9
void print_error(const T &t, Ts &&... ts)
Print items to std::cerr (items separated by spaces) and terminate with a new line.
Definition print.h:248
std::ptrdiff_t rel_fn_ptr_t
Definition worldrmi.h:105
ThreadBinder binder
Definition thread.cc:71
static void myusleep(unsigned int us)
Sleep or spin for specified number of microseconds.
Definition timers.h:186
std::shared_ptr< FunctionFunctorInterface< double, 3 > > func(new opT(g))
void(* rmi_handlerT)(void *buf, size_t nbyte)
This is the generic low-level interface for a message handler.
Definition worldrmi.h:104
static const double b
Definition nonlinschro.cc:119
static const double a
Definition nonlinschro.cc:118
Defines simple templates for printing to std::cout "a la Python".
Serializes calls to MPI in case it does not support THREAD_MULTIPLE.
This for RMI server thread to manage lifetime of WorldAM messages that it is sending.
Definition worldrmi.h:159
virtual bool TestAndFree()=0
virtual ~RMISendReq()
Definition worldrmi.h:161
Definition worldrmi.h:147
uint64_t nbyte_recv
Definition worldrmi.h:151
uint64_t nbyte_sent
Definition worldrmi.h:149
uint64_t nmsg_recv
Definition worldrmi.h:150
RMIStats()
Definition worldrmi.h:154
uint64_t nmsg_sent
Definition worldrmi.h:148
uint64_t max_serv_send_q
Definition worldrmi.h:152
Definition worldrmi.h:214
attrT attr
Definition worldrmi.h:216
rel_fn_ptr_t func
Definition worldrmi.h:215
rmi_handlerT func
Definition worldrmi.h:111
uint32_t attrT
attributes of the message; high 16 bits are the counter
Definition worldrmi.h:109
uint16_t counterT
counter for ordered messages
Definition worldrmi.h:108
friend bool operator<(const qmsg &a, const qmsg &b)
Definition worldrmi.h:124
qmsg()
Definition worldrmi.h:142
size_t len
Definition worldrmi.h:110
qmsg(size_t len, rmi_handlerT func, int i, int src, attrT attr, counterT count)
Definition worldrmi.h:117
attrT attr
Definition worldrmi.h:114
ProcessID src
Definition worldrmi.h:113
int i
Definition worldrmi.h:112
counterT count
Definition worldrmi.h:115
const char * status[2]
Definition testperiodic.cc:43
Implements Dqueue, Thread, ThreadBase and ThreadPool.
Defines types used by the parallel runtime.
int ProcessID
Used to clearly identify process number/rank.
Definition worldtypes.h:43