MADNESS 0.10.1
parallel_dc_archive.h
Go to the documentation of this file.
1#ifndef MAD_PARALLEL_DC_ARCHIVE_H_INCLUDED
2#define MAD_PARALLEL_DC_ARCHIVE_H_INCLUDED
3
8
9#include <algorithm> // std::min
10#include <cstdint> // std::uint64_t
11#include <cstring> // std::memcpy
12#include <limits> // std::numeric_limits
13
14namespace madness {
15
16
17
18 namespace archive {
19
20 /// Tag byte that starts every framed record (see ContainerRecordOutputArchive::close).
21 ///
22 /// A record larger than the RMI INT_MAX message limit cannot be sent through
23 /// `WorldContainer::replace` in one piece, so it is split: the record at `key`
24 /// becomes a header `[RECORD_CHUNKED][total][nchunk]` and the payload is stored as
25 /// `nchunk` records `[RECORD_CHUNK][key][i][slice]` at `container_record_chunk_key(key, i)`.
26 /// Any other record is `[RECORD_INLINE][payload]`, so the tag alone tells the reader
27 /// which it is.
28 ///
29 /// Chunk keys share the container's key space, so a chunk can still collide with an
30 /// unrelated record. That is unlikely (64-bit keys) but not prevented: ruling it out
31 /// would need an existence check on the remote owner, and WorldContainer's only remote
32 /// lookup, find(), copies the value, which may itself exceed 2 GiB. A collision is
33 /// detected instead: each chunk names its record and index, so the reader throws
34 /// rather than returning another record's bytes.
35 ///
36 /// Records are write-once: a key holding a chunked record must be freed with
37 /// erase_container_record() before it is written again, or the old chunks stay
38 /// allocated (not checked, for the same reason). The cloud never rewrites a key.
39 ///
40 /// An unframed record (`framed=false`) is the bare payload, never chunked: for
41 /// containers whose bytes are read directly rather than through
42 /// ContainerRecordInputArchive (the cloud's batch container, which has its own
43 /// chunked transport). Framed and unframed records must not share a container.
44 enum ContainerRecordTag : unsigned char {
48 };
49
50 /// Largest payload sent through a single WorldContainer::replace.
51 ///
52 /// The worldrmi assert is on the *total* message size (serialized key + value +
53 /// AM header), so we leave 1 MiB of headroom below INT_MAX for that overhead.
54 static constexpr std::size_t CONTAINER_RECORD_MAX_CHUNK =
55 static_cast<std::size_t>(std::numeric_limits<int>::max()) - (std::size_t(1) << 20);
56
57 /// Key of chunk `ichunk` of the chunked record at `key`: hashed, not `key+1, key+2, …`,
58 /// so a chunk cannot land on a neighbouring record whose key a caller chose sequentially.
59 ///
60 /// A full 64-bit mix (splitmix64's finalizer, a bijection), not worldhash's hash_value,
61 /// which folds a long to 32 bits: two records whose keys agreed in those 32 bits would
62 /// get identical chunk keys.
63 inline long container_record_chunk_key(const long key, const std::uint64_t ichunk) {
64 auto mix = [](std::uint64_t z) {
65 z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ull;
66 z = (z ^ (z >> 27)) * 0x94d049bb133111ebull;
67 return z ^ (z >> 31);
68 };
69 return long(mix(mix(std::uint64_t(key) ^ 0x4D41444E43484B53ull) + ichunk)); // "MADNCHKS"
70 }
71
72 /// size of a chunk's prefix `[RECORD_CHUNK][key][ichunk]`
73 static constexpr std::size_t CONTAINER_RECORD_CHUNK_PREFIX = 1 + sizeof(long) + sizeof(std::uint64_t);
74
75 /// parse a chunk header; false if `raw` is not one
76 ///
77 /// Not in a nested `detail` namespace: code that says `using namespace madness;
78 /// using namespace archive;` would then find `detail::` ambiguous.
79 inline bool parse_container_record_chunk_header(const std::vector<unsigned char>& raw,
80 std::uint64_t& total, std::uint64_t& nchunk) {
81 if (raw.size() != 1 + 2 * sizeof(std::uint64_t) or raw[0] != RECORD_CHUNKED) return false;
82 std::memcpy(&total, raw.data() + 1, sizeof(total));
83 std::memcpy(&nchunk, raw.data() + 1 + sizeof(total), sizeof(nchunk));
84 return true;
85 }
86
87 /// Erase a framed record and, if it is chunked, its chunks; erasing only `key` would
88 /// leave the chunks allocated.
89 inline void erase_container_record(WorldContainer<long, std::vector<unsigned char>>& dc,
90 const long key) {
91 auto it = dc.find(key).get();
92 if (it == dc.end()) return;
93 std::uint64_t total = 0, nchunk = 0;
94 const bool chunked = parse_container_record_chunk_header(it->second, total, nchunk);
95 dc.erase(key);
96 if (chunked)
97 for (std::uint64_t c = 0; c < nchunk; ++c) dc.erase(container_record_chunk_key(key, c));
98 }
99
101 public:
102 using keyT = long;
103 private:
107 containerT& dc; // lifetime???
108 const bool framed;
109 bool closed = false; // the destructor closes too; write the record only once
110 std::vector<unsigned char> v;
112
113 public:
114
115 /// @param[in] framed false writes the bare payload, never chunked (see ContainerRecordTag)
117 const bool framed = true)
119 , key(key)
120 , dc(dc)
121 , framed(framed)
122 , v()
123 , ar(v)
124 {
125 if (framed and subworld.rank() == 0) {
126 const unsigned char tag = RECORD_INLINE;
127 ar.store(&tag, 1);
128 }
129 }
130
135
137 return ar;
138 }
139 template <class T>
140 inline
141 typename std::enable_if< madness::is_trivially_serializable<T>::value, void >::type
142 store(const T* t, long n) const {
143 MADNESS_CHECK(subworld.rank() == 0);
144 ar.store(t, n);
145 }
146
147 void open() {}
148
149 void flush() {}
150
151 void close() {
152 if (closed) return;
153 closed = true;
154 if (subworld.rank() != 0) return;
155
156 // common case: the serialized record fits in a single RMI message
157 if (not framed or v.size() <= CONTAINER_RECORD_MAX_CHUNK) {
158 dc.replace(key, v);
159 return;
160 }
161
162 // oversized record: write a header at `key` and split the payload (which
163 // follows the inline tag at v[0]) into tagged chunks (see ContainerRecordTag).
164 const std::uint64_t total = v.size() - 1;
166 const std::uint64_t nchunk = (total + slice - 1) / slice;
167 std::vector<unsigned char> header(1 + 2 * sizeof(std::uint64_t));
168 header[0] = RECORD_CHUNKED;
169 std::memcpy(header.data() + 1, &total, sizeof(total));
170 std::memcpy(header.data() + 1 + sizeof(total), &nchunk, sizeof(nchunk));
171 dc.replace(key, header);
172
173 for (std::uint64_t c = 0; c < nchunk; ++c) {
174 const std::size_t start = 1 + c * slice;
175 const std::size_t end = std::min<std::size_t>(start + slice, v.size());
176 std::vector<unsigned char> chunk(CONTAINER_RECORD_CHUNK_PREFIX);
177 chunk[0] = RECORD_CHUNK;
178 std::memcpy(chunk.data() + 1, &key, sizeof(key));
179 std::memcpy(chunk.data() + 1 + sizeof(key), &c, sizeof(c));
180 chunk.insert(chunk.end(), v.begin() + start, v.begin() + end);
181 dc.replace(container_record_chunk_key(key, c), chunk);
182 }
183 }
184 };
185
187 using keyT = long;
190 std::vector<unsigned char> v;
192
193 // reassemble a chunked record: concatenate the payloads of its `nchunk` chunks
194 // (see ContainerRecordTag).
195 static std::vector<unsigned char>
196 load_chunks(const containerT& dc, const keyT& key, std::uint64_t total, std::uint64_t nchunk) {
197 std::vector<unsigned char> buf;
198 buf.reserve(total);
199 for (std::uint64_t c = 0; c < nchunk; ++c) {
201 MADNESS_CHECK_THROW(it != dc.end(), "ContainerRecordInputArchive: chunk record not found");
202 const std::vector<unsigned char>& chunk = it->second;
203 keyT parent = 0;
204 std::uint64_t index = 0;
205 const bool is_chunk = chunk.size() >= CONTAINER_RECORD_CHUNK_PREFIX and chunk[0] == RECORD_CHUNK;
206 if (is_chunk) {
207 std::memcpy(&parent, chunk.data() + 1, sizeof(parent));
208 std::memcpy(&index, chunk.data() + 1 + sizeof(parent), sizeof(index));
209 }
210 MADNESS_CHECK_THROW(is_chunk and parent == key and index == c,
211 "ContainerRecordInputArchive: chunk overwritten by another record (key collision)");
212 buf.insert(buf.end(), chunk.begin() + CONTAINER_RECORD_CHUNK_PREFIX, chunk.end());
213 }
214 MADNESS_CHECK_THROW(buf.size() == total,
215 "ContainerRecordInputArchive: chunked record size mismatch");
216 return buf;
217 }
218
219 // fetch the record at `key`, transparently reassembling chunked records
220 static std::vector<unsigned char>
221 load_raw(const World& subworld, const containerT& dc, const keyT& key) {
222 containerT::const_iterator it = dc.find(key).get();
223 if (it == dc.end()) {
224 std::cout << "key " << key << " in world " << subworld.id()
225 << "dc.world " << dc.get_world().id() << std::endl;
226 MADNESS_EXCEPTION("record not found", key);
227 }
228 const std::vector<unsigned char>& raw = it->second;
229
230 // the tag byte says whether this is the record itself or a chunk header
231 if (not raw.empty() and raw[0] == RECORD_INLINE)
232 return std::vector<unsigned char>(raw.begin() + 1, raw.end());
233 std::uint64_t total = 0, nchunk = 0;
235 "ContainerRecordInputArchive: untagged record (written with framed=false?)");
236 return load_chunks(dc, key, total, nchunk);
237 }
238
239 public:
240 ContainerRecordInputArchive(World& subworld, const containerT& dc, const keyT& key)
241 : rank(subworld.rank())
242 , v(rank == 0 ? load_raw(subworld, dc, key) : std::vector<unsigned char>{})
243 , ar(v)
244 {}
245
248
249 template <class T>
250 inline
251 typename std::enable_if< madness::is_trivially_serializable<T>::value, void >::type
252 load(T* t, long n) const {
253 MADNESS_CHECK(rank == 0);
254 ar.load(t,n);
255 }
256
257 void open() {}
258
259 void flush() {}
260
261 void close() {}
262 };
263
264
265 /// Implementation of functions for storing the pre/postamble in ContainerRecord archives.
266
267 /// \attention No type checking over Vector buffers, for efficiency.
268 /// \tparam T The data type.
269 template <class T>
271 /// Store the preamble.
272
273 /// \param[in] ar The archive.
275
276 /// Store the postamble.
277
278 /// \param[in] ar The archive.
279 static inline void postamble_store(const ContainerRecordOutputArchive& ar) {};
280 };
281
282 /// Implementation of functions for loading the pre/postamble in ContainerRecord archives.
283
284 /// \attention No type checking over ContainerRecord buffers, for efficiency.
285 /// \tparam T The data type.
286 template <class T>
288 /// Load the preamble.
289
290 /// \param[in] ar The archive.
291 static inline void preamble_load(const ContainerRecordInputArchive& ar) {};
292
293 /// Load the postamble.
294
295 /// \param[in] ar The archive.
296 static inline void postamble_load(const ContainerRecordInputArchive& ar) {};
297 };
298
299 // Forward storing to VectorOutputArchive
300 template <class keyT, class valueT>
303 std::vector<unsigned char> v;
304 VectorOutputArchive dummyar(v,0);
305 const int me = ar.get_world()->rank();
306
307 // Need to pass local archive by reference
308 ParallelOutputArchive<VectorOutputArchive> par(*(ar.get_world()), (me==0) ? ar.local_archive().get_archive() : dummyar);
309 par & t;
310
311 }
312 };
313
314
315
316 }
317
318
319}
320
321#endif
This header should include pretty much everything needed for the parallel runtime.
Makes a distributed container with specified attributes.
Definition worlddc.h:1302
bool find(accessor &acc, const keyT &key)
Write access to LOCAL value by key. Returns true if found, false otherwise (always false for remote).
Definition worlddc.h:1469
implT::const_iterator const_iterator
Definition worlddc.h:1310
iterator end()
Returns an iterator past the end of the local data (no communication).
Definition worlddc.h:1566
World & get_world() const
Returns the world associated with this container.
Definition worlddc.h:1411
A parallel world class.
Definition world.h:134
ProcessID rank() const
Returns the process rank in this World (same as MPI_Comm_rank()).
Definition world.h:344
unsigned long id() const
Definition world.h:324
Base class for input archive classes.
Definition archive.h:375
Base class for output archive classes.
Definition archive.h:383
World * get_world() const
Returns a pointer to the world.
Definition parallel_archive.h:130
Archive & local_archive() const
Returns a reference to the local archive.
Definition parallel_archive.h:248
Definition parallel_dc_archive.h:186
static std::vector< unsigned char > load_chunks(const containerT &dc, const keyT &key, std::uint64_t total, std::uint64_t nchunk)
Definition parallel_dc_archive.h:196
ProcessID rank
Definition parallel_dc_archive.h:189
void close()
Definition parallel_dc_archive.h:261
long keyT
Definition parallel_dc_archive.h:187
WorldContainer< keyT, std::vector< unsigned char > > containerT
Definition parallel_dc_archive.h:188
void open()
Definition parallel_dc_archive.h:257
std::vector< unsigned char > v
Definition parallel_dc_archive.h:190
ContainerRecordInputArchive(World &subworld, const containerT &dc, const keyT &key)
Definition parallel_dc_archive.h:240
void flush()
Definition parallel_dc_archive.h:259
VectorInputArchive ar
Definition parallel_dc_archive.h:191
std::enable_if< madness::is_trivially_serializable< T >::value, void >::type load(T *t, long n) const
Definition parallel_dc_archive.h:252
~ContainerRecordInputArchive()
Definition parallel_dc_archive.h:246
static std::vector< unsigned char > load_raw(const World &subworld, const containerT &dc, const keyT &key)
Definition parallel_dc_archive.h:221
Definition parallel_dc_archive.h:100
const bool framed
Definition parallel_dc_archive.h:108
WorldContainer< keyT, std::vector< unsigned char > > containerT
Definition parallel_dc_archive.h:104
std::vector< unsigned char > v
Definition parallel_dc_archive.h:110
void open()
Definition parallel_dc_archive.h:147
VectorOutputArchive & get_archive()
Definition parallel_dc_archive.h:136
void flush()
Definition parallel_dc_archive.h:149
bool closed
Definition parallel_dc_archive.h:109
ContainerRecordOutputArchive(World &subworld, containerT &dc, const keyT &key, const bool framed=true)
Definition parallel_dc_archive.h:116
VectorOutputArchive ar
Definition parallel_dc_archive.h:111
~ContainerRecordOutputArchive()
Definition parallel_dc_archive.h:131
keyT key
Definition parallel_dc_archive.h:106
World & subworld
Definition parallel_dc_archive.h:105
long keyT
Definition parallel_dc_archive.h:102
containerT & dc
Definition parallel_dc_archive.h:107
void close()
Definition parallel_dc_archive.h:151
std::enable_if< madness::is_trivially_serializable< T >::value, void >::type store(const T *t, long n) const
Definition parallel_dc_archive.h:142
An archive for storing local or parallel data wrapping a BinaryFstreamOutputArchive.
Definition parallel_archive.h:321
Wraps an archive around an STL vector for input.
Definition vector_archive.h:101
Wraps an archive around an STL vector for output.
Definition vector_archive.h:55
static const double v
Definition hatom_sf_dirac.cc:20
#define mix(a, b, c)
Definition lookup3.c:130
#define MADNESS_CHECK(condition)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:182
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_CHECK_THROW(condition, msg)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:207
Namespace for I/O tools.
Definition madness.h:49
static constexpr std::size_t CONTAINER_RECORD_CHUNK_PREFIX
size of a chunk's prefix [RECORD_CHUNK][key][ichunk]
Definition parallel_dc_archive.h:73
static constexpr std::size_t CONTAINER_RECORD_MAX_CHUNK
Definition parallel_dc_archive.h:54
long container_record_chunk_key(const long key, const std::uint64_t ichunk)
Definition parallel_dc_archive.h:63
bool parse_container_record_chunk_header(const std::vector< unsigned char > &raw, std::uint64_t &total, std::uint64_t &nchunk)
Definition parallel_dc_archive.h:79
void erase_container_record(WorldContainer< long, std::vector< unsigned char > > &dc, const long key)
Definition parallel_dc_archive.h:89
ContainerRecordTag
Definition parallel_dc_archive.h:44
@ RECORD_CHUNKED
Definition parallel_dc_archive.h:46
@ RECORD_INLINE
Definition parallel_dc_archive.h:45
@ RECORD_CHUNK
Definition parallel_dc_archive.h:47
Namespace for all elements and tools of MADNESS.
Definition DFConvergence.h:9
Definition mraimpl.h:53
static const double c
Definition relops.cc:10
static void postamble_load(const ContainerRecordInputArchive &ar)
Load the postamble.
Definition parallel_dc_archive.h:296
static void preamble_load(const ContainerRecordInputArchive &ar)
Load the preamble.
Definition parallel_dc_archive.h:291
static void postamble_store(const ContainerRecordOutputArchive &ar)
Store the postamble.
Definition parallel_dc_archive.h:279
static void preamble_store(const ContainerRecordOutputArchive &ar)
Store the preamble.
Definition parallel_dc_archive.h:274
Default implementation of the pre/postamble for type checking.
Definition archive.h:510
static void store(const ParallelOutputArchive< ContainerRecordOutputArchive > &ar, const WorldContainer< keyT, valueT > &t)
Definition parallel_dc_archive.h:302
Default store of an object via serialize(ar, t).
Definition archive.h:612
std::pair< int, double > valueT
Definition test_binsorter.cc:6
int me
Definition test_binsorter.cc:10
Implements an archive wrapping an STL vector.
Implements WorldContainer.
Defines hash functions for use in distributed containers.
int ProcessID
Used to clearly identify process number/rank.
Definition worldtypes.h:43
Key< D > keyT
Definition writecoeff2.cc:11