MADNESS 0.10.1
test_rmi_huge.h
Go to the documentation of this file.
1/*
2 This file is part of MADNESS.
3
4 Copyright (C) 2007,2010 Oak Ridge National Laboratory
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19*/
20
21/// \file test_rmi_huge.h
22/// \brief Messages too large for an MPI count of bytes, sent as the argument of a remote task.
23///
24/// A message longer than \c MAD_BUFFER_SIZE takes RMI's huge-message protocol, and one of
25/// more than \c INT_MAX bytes must further be given to MPI either by its large-count
26/// functions (MPI 4) or, where MPI has none, described by a derived datatype
27/// ( \c SafeMPI::ByteCount ). This is the body of two tests, each a source file that
28/// includes it:
29/// - test_rmi_huge.cc lowers the size above which a plain count of bytes is not used,
30/// so that messages of a few MiB cover the other paths: one that leaves a remainder
31/// after whole chunks of the derived datatype, one that does not, one that stays a
32/// plain count, and several at once. Where MPI has large counts these are sent that
33/// way, then by the derived datatype, then with the two ends disagreeing. It also
34/// checks the datatypes ByteCount builds for sizes beyond \c INT_MAX , which takes no
35/// memory.
36/// - test_rmi_huge_large.cc defines \c MADNESS_TEST_RMI_HUGE_LARGE and sends one message
37/// of more than \c INT_MAX bytes at the default setting (and, where MPI has large
38/// counts, once more by the derived datatype); that takes about 9 GiB, so it carries
39/// the "large" label.
40///
41/// Needs at least 2 ranks to send anything; with 1 the tasks are local.
42
43#ifndef MADNESS_WORLD_TEST_RMI_HUGE_H__INCLUDED
44#define MADNESS_WORLD_TEST_RMI_HUGE_H__INCLUDED
45
46#include <cstdint>
47#include <cstdlib>
48#include <iostream>
49#include <limits>
50#include <string>
51#include <vector>
52
55
56using namespace madness;
57
58namespace {
59
60/// The payload is words rather than bytes: serialization counts the elements of an array
61/// in an unsigned int, so a vector of bytes could not test a message of 4 GiB or more.
62using wordT = std::uint64_t;
63
64/// the word at position \c i of the message with the given \c seed
65inline wordT pattern(std::size_t i, std::uint64_t seed) {
66 return (i + seed) * 0x9E3779B97F4A7C15ull;
67}
68
69/// \param nbyte the size of the message, rounded up to whole words
70std::vector<wordT> make_message(std::size_t nbyte, std::uint64_t seed) {
71 std::vector<wordT> v((nbyte + sizeof(wordT) - 1) / sizeof(wordT));
72 for (std::size_t i = 0; i < v.size(); ++i) v[i] = pattern(i, seed);
73 return v;
74}
75
76/// runs on the receiver: the number of words that differ from the pattern, or
77/// \c size_t(-1) if the size is wrong
78std::size_t count_errors(const std::vector<wordT>& v, std::size_t nbyte, std::uint64_t seed) {
79 if (v.size() != (nbyte + sizeof(wordT) - 1) / sizeof(wordT)) return std::size_t(-1);
80 std::size_t nerr = 0;
81 for (std::size_t i = 0; i < v.size(); ++i) nerr += (v[i] != pattern(i, seed));
82 return nerr;
83}
84
85/// sends messages of the given sizes to the next rank, all pending at once
86bool send_and_check(World& world, const std::vector<std::size_t>& sizes, const std::string& what) {
87 const ProcessID dest = (world.rank() + 1) % world.size();
88 std::vector<Future<std::size_t>> results;
89 for (std::size_t k = 0; k < sizes.size(); ++k) {
90 const std::uint64_t seed = 1000 * world.rank() + k;
91 results.push_back(world.taskq.add(dest, &count_errors, make_message(sizes[k], seed), sizes[k], seed));
92 }
93 world.gop.fence();
94 bool ok = true;
95 for (std::size_t k = 0; k < sizes.size(); ++k) {
96 const std::size_t nerr = results[k].get();
97 if (nerr != 0) {
98 ok = false;
99 std::cout << "rank " << world.rank() << ": " << what << ": message of " << sizes[k]
100 << " bytes arrived with " << (nerr == std::size_t(-1) ? "the wrong size" : "wrong contents")
101 << std::endl;
102 }
103 }
104 int all_ok = ok;
105 world.gop.logic_and(&all_ok, 1);
106 if (world.rank() == 0) std::cout << what << ": " << (all_ok ? "ok" : "FAILED") << std::endl;
107 return all_ok;
108}
109
110#ifndef STUBOUTMPI
111/// whether \c bc describes \c n contiguous bytes by a derived datatype
112bool describes(const SafeMPI::ByteCount& bc, std::size_t n) {
113 if (!bc.derived() || bc.count() != 1) return false;
114 // contiguous bytes from 0: the extent, with and without padding, is the size
115 MPI_Aint lb = -1, extent = -1, true_lb = -1, true_extent = -1;
116 MPI_Type_get_extent(bc.datatype(), &lb, &extent);
117 MPI_Type_get_true_extent(bc.datatype(), &true_lb, &true_extent);
118 return lb == 0 && true_lb == 0 && static_cast<std::size_t>(extent) == n &&
119 static_cast<std::size_t>(true_extent) == n;
120}
121
122/// SafeMPI::ByteCount itself, with no message and no buffer: which sizes take a derived
123/// datatype, and what it describes, sizes beyond \c INT_MAX at the default chunk included
124bool test_byte_count() {
125 const std::size_t int_max = static_cast<std::size_t>(std::numeric_limits<int>::max());
126 bool ok = true;
127 {
128 SafeMPI::ByteCount bc(1000, 1000);
129 ok = ok && !bc.derived() && bc.count() == 1000 && bc.datatype() == MPI_BYTE;
130 }
131 {
132 SafeMPI::ByteCount bc(int_max);
133 ok = ok && !bc.derived() && bc.count() == std::numeric_limits<int>::max();
134 }
135 for (std::size_t n : {std::size_t(1001), std::size_t(2000), std::size_t(123457)}) {
136 SafeMPI::ByteCount bc(n, 1000);
137 ok = ok && describes(bc, n);
138 }
139 for (std::size_t n : {int_max + 1, 2 * int_max, 2 * int_max + 1, (std::size_t(1) << 31) + (std::size_t(1) << 20) + 7,
140 std::size_t(5) << 32, (std::size_t(1) << 40) + 3}) {
142 ok = ok && describes(bc, n);
143 }
144 return ok;
145}
146#endif
147
148} // namespace
149
150int main(int argc, char** argv) {
151 World& world = initialize(argc, argv);
152 bool ok = true;
153
154#ifdef MADNESS_TEST_RMI_HUGE_LARGE
155 if (world.size() == 1) {
156 std::cout << "a message of more than INT_MAX bytes: skipped, needs 2 ranks" << std::endl;
157 finalize();
158 return 0;
159 }
160 // one message of more than INT_MAX bytes, from rank 0 only: each one is held four
161 // times over (the vector, its serialized form, the receive buffer, the vector again)
162 // MAD_TEST_RMI_HUGE_BYTES overrides the size, to try others by hand
163 std::size_t n = (std::size_t(1) << 31) + (std::size_t(1) << 20) + 7;
164 if (const char* env = std::getenv("MAD_TEST_RMI_HUGE_BYTES")) n = std::strtoull(env, nullptr, 10);
165 ok = send_and_check(world, world.rank() == 0 ? std::vector<std::size_t>{n} : std::vector<std::size_t>{},
166 std::string("a message of more than INT_MAX bytes") +
167 (SafeMPI::ByteCount::use_large_count() ? " (large count)" : " (derived datatype)")) && ok;
169 // again with what an MPI without large counts would do
171 ok = send_and_check(world, world.rank() == 0 ? std::vector<std::size_t>{n} : std::vector<std::size_t>{},
172 "a message of more than INT_MAX bytes (derived datatype)") && ok;
174 }
175#else
176 const std::size_t MiB = std::size_t(1) << 20;
177 // a huge message in RMI's terms, whatever MAD_BUFFER_SIZE is; with one rank there is
178 // no RMI to ask, and nothing is sent
179 const std::size_t huge = (world.size() > 1 ? RMI::max_msg_len() : MiB) + 3 * MiB;
180
181#ifndef STUBOUTMPI
182 if (world.rank() == 0) {
183 const bool bc_ok = test_byte_count();
184 std::cout << "ByteCount: " << (bc_ok ? "ok" : "FAILED") << std::endl;
185 ok = bc_ok && ok;
186 }
187#endif
188
189 ok = send_and_check(world, {huge + 12345}, "a plain count") && ok;
190
191 // above the plain count MPI is given either a large count (MPI 4) or a derived
192 // datatype; where MPI has large counts both are tried, and then also one on each end
193 std::vector<bool> large_count_choices{false};
194 if (SafeMPI::ByteCount::have_large_count()) large_count_choices.insert(large_count_choices.begin(), true);
195 for (const bool large_count : large_count_choices) {
197 const std::string how = large_count ? " (large count)" : " (derived datatype)";
199 ok = send_and_check(world, {huge + 12345}, "one message, not a whole number of chunks" + how) && ok;
200 // the serialized task is larger than its payload, so pick the datatype's chunk to
201 // divide the message rather than the payload: a chunk of 1 byte always does
203 ok = send_and_check(world, {huge / 8}, "one message, a whole number of chunks" + how) && ok;
205 ok = send_and_check(world, {huge + 1, 2 * huge + 77, huge + MiB, 100, huge + 5}, "several at once" + how) && ok;
206 }
208 // the two ends need not describe a message the same way
210 ok = send_and_check(world, {huge + 12345, 2 * huge + 77}, "large count on one end, derived datatype on the other") && ok;
211 }
214#endif
215
216 world.gop.fence();
217 finalize();
218 if (!ok) return 1;
219 return 0;
220}
221
222#endif // MADNESS_WORLD_TEST_RMI_HUGE_H__INCLUDED
This header should include pretty much everything needed for the parallel runtime.
The (count, datatype) pair that describes a buffer of bytes to MPI.
Definition safempi.h:531
static bool use_large_count()
Definition safempi.h:619
static void set_use_large_count(bool flag)
Sets use_large_count().
Definition safempi.h:626
static constexpr bool have_large_count()
Definition safempi.h:608
static void set_max_plain_count(std::size_t n)
Sets max_plain_count().
Definition safempi.h:601
static std::size_t max_msg_len()
Returns the size of recv buffers, in bytes.
Definition worldrmi.h:329
void fence(bool debug=false)
Synchronizes all processes in communicator AND globally ensures no pending AM or tasks.
Definition worldgop.cc:177
void logic_and(T *buf, size_t nelem)
Definition worldgop.h:940
void add(TaskInterface *t)
Add a new local task, taking ownership of the pointer.
Definition world_task_queue.h:466
A parallel world class.
Definition world.h:134
WorldTaskQueue & taskq
Task queue.
Definition world.h:215
ProcessID rank() const
Returns the process rank in this World (same as MPI_Comm_rank()).
Definition world.h:344
ProcessID size() const
Returns the number of processes in this World (same as MPI_Comm_size()).
Definition world.h:354
WorldGopInterface & gop
Global operations.
Definition world.h:216
static const double v
Definition hatom_sf_dirac.cc:20
Namespace for all elements and tools of MADNESS.
Definition DFConvergence.h:9
std::optional< BoundaryConditions< NDIM > > FunctionDefaults< NDIM >::bc
Definition mraimpl.h:3890
void finalize()
Call this once at the very end of your main program instead of MPI_Finalize().
Definition world.cc:246
World & initialize(int &argc, char **&argv, bool quiet)
Definition world.cc:153
static const long k
Definition rk.cc:44
Serializes calls to MPI in case it does not support THREAD_MULTIPLE.
std::ptrdiff_t MPI_Aint
Definition stubmpi.h:32
int MPI_Type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent)
Definition stubmpi.h:99
#define MPI_BYTE
Definition stubmpi.h:78
std::string ok(const bool b)
Definition test6.cc:48
int main()
Definition test_workflow_builders.cpp:14
int ProcessID
Used to clearly identify process number/rank.
Definition worldtypes.h:43