1#ifndef MAD_PARALLEL_DC_ARCHIVE_H_INCLUDED
2#define MAD_PARALLEL_DC_ARCHIVE_H_INCLUDED
55 static_cast<std::size_t
>(std::numeric_limits<int>::max()) - (std::size_t(1) << 20);
64 auto mix = [](std::uint64_t z) {
65 z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ull;
66 z = (z ^ (z >> 27)) * 0x94d049bb133111ebull;
69 return long(
mix(
mix(std::uint64_t(key) ^ 0x4D41444E43484B53ull) + ichunk));
80 std::uint64_t& total, std::uint64_t& nchunk) {
81 if (raw.size() != 1 + 2 *
sizeof(std::uint64_t) or raw[0] !=
RECORD_CHUNKED)
return false;
82 std::memcpy(&total, raw.data() + 1,
sizeof(total));
83 std::memcpy(&nchunk, raw.data() + 1 +
sizeof(total),
sizeof(nchunk));
91 auto it = dc.find(key).get();
92 if (it == dc.end())
return;
93 std::uint64_t total = 0, nchunk = 0;
110 std::vector<unsigned char>
v;
126 const unsigned char tag = RECORD_INLINE;
141 typename std::enable_if< madness::is_trivially_serializable<T>::value,
void >
::type
164 const std::uint64_t total =
v.size() - 1;
166 const std::uint64_t nchunk = (total + slice - 1) / slice;
167 std::vector<unsigned char> header(1 + 2 *
sizeof(std::uint64_t));
169 std::memcpy(header.data() + 1, &total,
sizeof(total));
170 std::memcpy(header.data() + 1 +
sizeof(total), &nchunk,
sizeof(nchunk));
171 dc.replace(
key, header);
173 for (std::uint64_t
c = 0;
c < nchunk; ++
c) {
174 const std::size_t start = 1 +
c * slice;
175 const std::size_t end = std::min<std::size_t>(start + slice,
v.size());
178 std::memcpy(chunk.data() + 1, &
key,
sizeof(
key));
179 std::memcpy(chunk.data() + 1 +
sizeof(
key), &
c,
sizeof(
c));
180 chunk.insert(chunk.end(),
v.begin() + start,
v.begin() + end);
190 std::vector<unsigned char>
v;
195 static std::vector<unsigned char>
197 std::vector<unsigned char> buf;
199 for (std::uint64_t
c = 0;
c < nchunk; ++
c) {
202 const std::vector<unsigned char>& chunk = it->second;
204 std::uint64_t index = 0;
207 std::memcpy(&parent, chunk.data() + 1,
sizeof(parent));
208 std::memcpy(&index, chunk.data() + 1 +
sizeof(parent),
sizeof(index));
211 "ContainerRecordInputArchive: chunk overwritten by another record (key collision)");
215 "ContainerRecordInputArchive: chunked record size mismatch");
220 static std::vector<unsigned char>
223 if (it == dc.
end()) {
224 std::cout <<
"key " << key <<
" in world " << subworld.
id()
225 <<
"dc.world " << dc.
get_world().
id() << std::endl;
228 const std::vector<unsigned char>& raw = it->second;
232 return std::vector<unsigned char>(raw.begin() + 1, raw.end());
233 std::uint64_t total = 0, nchunk = 0;
235 "ContainerRecordInputArchive: untagged record (written with framed=false?)");
251 typename std::enable_if< madness::is_trivially_serializable<T>::value,
void >
::type
300 template <
class keyT,
class valueT>
303 std::vector<unsigned char>
v;
This header should include pretty much everything needed for the parallel runtime.
Makes a distributed container with specified attributes.
Definition worlddc.h:1302
bool find(accessor &acc, const keyT &key)
Write access to LOCAL value by key. Returns true if found, false otherwise (always false for remote).
Definition worlddc.h:1469
implT::const_iterator const_iterator
Definition worlddc.h:1310
iterator end()
Returns an iterator past the end of the local data (no communication).
Definition worlddc.h:1566
World & get_world() const
Returns the world associated with this container.
Definition worlddc.h:1411
A parallel world class.
Definition world.h:134
ProcessID rank() const
Returns the process rank in this World (same as MPI_Comm_rank()).
Definition world.h:344
unsigned long id() const
Definition world.h:324
Base class for output archive classes.
Definition archive.h:383
World * get_world() const
Returns a pointer to the world.
Definition parallel_archive.h:130
Archive & local_archive() const
Returns a reference to the local archive.
Definition parallel_archive.h:248
Definition parallel_dc_archive.h:100
const bool framed
Definition parallel_dc_archive.h:108
WorldContainer< keyT, std::vector< unsigned char > > containerT
Definition parallel_dc_archive.h:104
std::vector< unsigned char > v
Definition parallel_dc_archive.h:110
void open()
Definition parallel_dc_archive.h:147
VectorOutputArchive & get_archive()
Definition parallel_dc_archive.h:136
void flush()
Definition parallel_dc_archive.h:149
bool closed
Definition parallel_dc_archive.h:109
ContainerRecordOutputArchive(World &subworld, containerT &dc, const keyT &key, const bool framed=true)
Definition parallel_dc_archive.h:116
VectorOutputArchive ar
Definition parallel_dc_archive.h:111
~ContainerRecordOutputArchive()
Definition parallel_dc_archive.h:131
keyT key
Definition parallel_dc_archive.h:106
World & subworld
Definition parallel_dc_archive.h:105
long keyT
Definition parallel_dc_archive.h:102
containerT & dc
Definition parallel_dc_archive.h:107
void close()
Definition parallel_dc_archive.h:151
std::enable_if< madness::is_trivially_serializable< T >::value, void >::type store(const T *t, long n) const
Definition parallel_dc_archive.h:142
An archive for storing local or parallel data wrapping a BinaryFstreamOutputArchive.
Definition parallel_archive.h:321
Wraps an archive around an STL vector for output.
Definition vector_archive.h:55
static const double v
Definition hatom_sf_dirac.cc:20
#define mix(a, b, c)
Definition lookup3.c:130
#define MADNESS_CHECK(condition)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:182
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_CHECK_THROW(condition, msg)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:207
Namespace for I/O tools.
Definition madness.h:49
static constexpr std::size_t CONTAINER_RECORD_CHUNK_PREFIX
size of a chunk's prefix [RECORD_CHUNK][key][ichunk]
Definition parallel_dc_archive.h:73
static constexpr std::size_t CONTAINER_RECORD_MAX_CHUNK
Definition parallel_dc_archive.h:54
long container_record_chunk_key(const long key, const std::uint64_t ichunk)
Definition parallel_dc_archive.h:63
bool parse_container_record_chunk_header(const std::vector< unsigned char > &raw, std::uint64_t &total, std::uint64_t &nchunk)
Definition parallel_dc_archive.h:79
void erase_container_record(WorldContainer< long, std::vector< unsigned char > > &dc, const long key)
Definition parallel_dc_archive.h:89
ContainerRecordTag
Definition parallel_dc_archive.h:44
@ RECORD_CHUNKED
Definition parallel_dc_archive.h:46
@ RECORD_INLINE
Definition parallel_dc_archive.h:45
@ RECORD_CHUNK
Definition parallel_dc_archive.h:47
Namespace for all elements and tools of MADNESS.
Definition DFConvergence.h:9
static const double c
Definition relops.cc:10
static void postamble_load(const ContainerRecordInputArchive &ar)
Load the postamble.
Definition parallel_dc_archive.h:296
static void preamble_load(const ContainerRecordInputArchive &ar)
Load the preamble.
Definition parallel_dc_archive.h:291
static void postamble_store(const ContainerRecordOutputArchive &ar)
Store the postamble.
Definition parallel_dc_archive.h:279
static void preamble_store(const ContainerRecordOutputArchive &ar)
Store the preamble.
Definition parallel_dc_archive.h:274
Default implementation of the pre/postamble for type checking.
Definition archive.h:510
static void store(const ParallelOutputArchive< ContainerRecordOutputArchive > &ar, const WorldContainer< keyT, valueT > &t)
Definition parallel_dc_archive.h:302
Default store of an object via serialize(ar, t).
Definition archive.h:612
std::pair< int, double > valueT
Definition test_binsorter.cc:6
int me
Definition test_binsorter.cc:10
Implements an archive wrapping an STL vector.
Implements WorldContainer.
Defines hash functions for use in distributed containers.
int ProcessID
Used to clearly identify process number/rank.
Definition worldtypes.h:43
Key< D > keyT
Definition writecoeff2.cc:11