cloud class More...

#include <cloud.h>

Classes
struct	cloudtimer

struct	is_madness_function_vector

struct	is_madness_function_vector< std::vector< Function< T, NDIM > > >

struct	is_tuple

struct	is_tuple< std::tuple< T... > >

struct	is_vector

struct	is_vector< std::vector< Q > >

Public Types
typedef std::any	cached_objT

typedef std::map< keyT, cached_objT >	cacheT

template<typename T >
using	has_cloud_serialize = madness::meta::is_detected< member_cloud_serialize_t, T >

using	keyT = madness::archive::ContainerRecordOutputArchive::keyT

template<typename T >
using	member_cloud_serialize_t = decltype(std::declval< T >().cloud_store(std::declval< World & >(), std::declval< Cloud & >()))

typedef Recordlist< keyT >	recordlistT

using	valueT = std::vector< unsigned char >

Public Member Functions
	Cloud (madness::World &universe)

void	clear ()

void	clear_cache (World &subworld)

void	clear_timings ()

template<typename T >
std::enable_if< is_vector< T >::value, T >::type	do_load (World &world, recordlistT &recordlist) const

template<typename T >
std::enable_if<!is_vector< T >::value, T >::type	do_load (World &world, recordlistT &recordlist) const

template<typename T >
T	forward_load (madness::World &world, recordlistT &recordlist) const
	load a single object from the cloud, recordlist is consumed while loading elements

template<typename T >
T	load (madness::World &world, const recordlistT recordlist) const

template<typename T >
T	load_tuple (madness::World &world, recordlistT &recordlist) const

void	print_size (World &universe)

void	print_timings (World &universe) const

void	replicate (const std::size_t chunk_size=INT_MAX)

void	set_debug (bool value)

void	set_fence (bool value)

void	set_force_load_from_cache (bool value)

template<typename T >
recordlistT	store (madness::World &world, const T &source)

template<typename T >
recordlistT	store_other (madness::World &world, const std::vector< T > &source)

template<typename... Ts>
recordlistT	store_tuple (World &world, const std::tuple< Ts... > &input)
	store a tuple in multiple records

Private Types
template<typename T >
using	is_parallel_serializable_object = std::is_base_of< archive::ParallelSerializableObject, T >

template<typename T >
using	is_world_constructible = std::is_constructible< T, World & >

Private Member Functions
template<typename T >
T	allocator (World &world) const

template<typename T >
void	cache (madness::World &world, const T &obj, const keyT &record) const

bool	is_cached (const keyT &key) const

bool	is_in_container (const keyT &key) const
	checks if a (universe) container record is used

template<typename T >
T	load_from_cache (madness::World &world, const keyT &record) const
	load an object from the cache, record is unchanged

template<typename T >
recordlistT	store_other (madness::World &world, const T &source)

Private Attributes
std::atomic< long >	cache_reads =0l

std::atomic< long >	cache_stores =0l

cacheT	cached_objects

madness::WorldContainer< keyT, valueT >	container

bool	debug = false
	prints debug output

bool	dofence = true
	fences after load/store

bool	force_load_from_cache = false
	forces load from cache (mainly for debugging)

bool	is_replicated =false
	if contents of the container are replicated

recordlistT	local_list_of_container_keys

std::atomic< long >	reading_time =0l

std::atomic< long >	replication_time =0l

std::atomic< long >	writing_time =0l

std::atomic< long >	writing_time1 =0l

Detailed Description

cloud class

store and load data to/from the cloud into arbitrary worlds

Distributed data is always bound to a certain world. If it needs to be present in another world it can be serialized to the cloud and deserialized from there again. For an example see test_cloud.cc

Data is stored into a distributed container living in the universe. During storing a (replicated) list of records is returned that can be used to find the data in the container. If a combined object (a vector, tuple, etc) is stored a list of records will be generated. When loading the data from the world the record list will be used to deserialize all stored objects.

Note that there must be a fence after the destruction of subworld containers, as in:

create subworlds { dcT(subworld) do work } subworld.gop.fence();

Member Typedef Documentation

◆ cached_objT

typedef std::any madness::Cloud::cached_objT

◆ cacheT

typedef std::map<keyT, cached_objT> madness::Cloud::cacheT

◆ has_cloud_serialize

template<typename T >

using madness::Cloud::has_cloud_serialize = madness::meta::is_detected<member_cloud_serialize_t, T>

◆ is_parallel_serializable_object

template<typename T >

using madness::Cloud::is_parallel_serializable_object = std::is_base_of<archive::ParallelSerializableObject,T>

private

◆ is_world_constructible

template<typename T >

using madness::Cloud::is_world_constructible = std::is_constructible<T, World &>

private

◆ keyT

using madness::Cloud::keyT = madness::archive::ContainerRecordOutputArchive::keyT

◆ member_cloud_serialize_t

template<typename T >

using madness::Cloud::member_cloud_serialize_t = decltype(std::declval<T>().cloud_store(std::declval<World&>(), std::declval<Cloud&>()))

◆ recordlistT

typedef Recordlist<keyT> madness::Cloud::recordlistT

◆ valueT

using madness::Cloud::valueT = std::vector<unsigned char>

Constructor & Destructor Documentation

◆ Cloud()

madness::Cloud::Cloud ( madness::World & universe )

inline

Parameters

[in] universe the universe world

Member Function Documentation

◆ allocator()

template<typename T >

T madness::Cloud::allocator ( World & world ) const

inlineprivate

References T().

◆ cache()

template<typename T >

void madness::Cloud::cache	(	madness::World &	world,
		const T &	obj,
		const keyT &	record
	)		const

inlineprivate

References cached_objects.

◆ clear()

void madness::Cloud::clear ( )

inline

References container.

Referenced by madness::MacroTaskQ::run_all(), and test_custom_serialization().

◆ clear_cache()

void madness::Cloud::clear_cache ( World & subworld )

inline

References cached_objects, madness::WorldGopInterface::fence(), madness::World::gop, and local_list_of_container_keys.

Referenced by main(), madness::MacroTaskQ::run_all(), and simple_example().

◆ clear_timings()

void madness::Cloud::clear_timings ( )

inline

References cache_reads, cache_stores, reading_time, replication_time, writing_time, and writing_time1.

Referenced by main().

◆ do_load() [1/2]

template<typename T >

std::enable_if< is_vector< T >::value, T >::type madness::Cloud::do_load	(	World &	world,
		recordlistT &	recordlist
	)		const

inline

load a vector from the cloud, pop records from recordlist

Parameters

[in,out]	world	destination world
[in,out]	recordlist	list of records to load from (reduced by the first few elements)

References T(), and target().

◆ do_load() [2/2]

template<typename T >

std::enable_if<!is_vector< T >::value, T >::type madness::Cloud::do_load	(	World &	world,
		recordlistT &	recordlist
	)		const

inline

load a single object from the cloud, pop record from recordlist

Parameters

[in,out]	world	destination world
[in,out]	recordlist	list of records to load from (reduced by the first element)

References madness::cache, container, debug, force_load_from_cache, madness::World::id(), is_cached(), MADNESS_CHECK, madness::name(), madness::print(), T(), and target().

◆ forward_load()

template<typename T >

T madness::Cloud::forward_load	(	madness::World &	world,
		recordlistT &	recordlist
	)		const

inline

load a single object from the cloud, recordlist is consumed while loading elements

References T(), and target().

Referenced by madness::CCPair::cloud_load(), madness::CCIntermediatePotentials::cloud_load(), madness::Info::cloud_load(), custom_serialize_tester::cloud_load(), and load_tuple().

◆ is_cached()

bool madness::Cloud::is_cached ( const keyT & key ) const

inlineprivate

References cached_objects.

Referenced by do_load().

◆ is_in_container()

bool madness::Cloud::is_in_container ( const keyT & key ) const

inlineprivate

checks if a (universe) container record is used

currently implemented with a local copy of the recordlist, might be reimplemented with container.find(), which would include blocking communication.

References local_list_of_container_keys.

Referenced by store_other().

◆ load()

template<typename T >

T madness::Cloud::load	(	madness::World &	world,
		const recordlistT	recordlist
	)		const

inline

Parameters

[in]	world	the subworld the objects are loaded to
[in]	recordlist	the list of records where the objects are stored

References reading_time.

Referenced by chunk_example(), madness::MacroTask< taskT >::MacroTaskInternal::get_output(), main(), madness::MacroTask< taskT >::MacroTaskInternal::run(), simple_example(), test_custom_serialization(), and test_custom_worldobject().

◆ load_from_cache()

template<typename T >

T madness::Cloud::load_from_cache	(	madness::World &	world,
		const keyT &	record
	)		const

inlineprivate

load an object from the cache, record is unchanged

References cache_reads, cached_objects, debug, madness::World::id(), MADNESS_EXCEPTION, madness::name(), madness::print(), madness::World::rank(), T(), and target().

◆ load_tuple()

template<typename T >

T madness::Cloud::load_tuple	(	madness::World &	world,
		recordlistT &	recordlist
	)		const

inline

load a tuple from the cloud, pop records from recordlist

Parameters

[in,out]	world	destination world
[in,out]	recordlist	list of records to load from (reduced by the first few elements)

References debug, forward_load(), madness::World::id(), madness::name(), T(), target(), and madness::type().

◆ print_size()

void madness::Cloud::print_size ( World & universe )

inline

References container, madness::World::gop, is_replicated, madness::WorldGopInterface::max(), madness::WorldGopInterface::min(), madness::print(), madness::World::rank(), madness::World::size(), and madness::WorldGopInterface::sum().

Referenced by madness::MacroTaskQ::run_all().

◆ print_timings()

void madness::Cloud::print_timings ( World & universe ) const

inline

References cache_reads, cache_stores, madness::World::gop, madness::print(), madness::World::rank(), reading_time, replication_time, madness::WorldGopInterface::sum(), and writing_time.

Referenced by main(), and madness::MacroTaskQ::run_all().

◆ replicate()

void madness::Cloud::replicate ( const std::size_t chunk_size = INT_MAX )

inline

References madness::WorldMpiInterface::Bcast(), container, madness::WorldGopInterface::fence(), madness::WorldContainer< keyT, valueT, hashfunT >::find(), madness::World::gop, is_replicated, MADNESS_CHECK, madness::World::mpi, MPI_BYTE, madness::World::rank(), replication_time, and madness::World::size().

Referenced by chunk_example(), and madness::MacroTaskQ::run_all().

◆ set_debug()

void madness::Cloud::set_debug ( bool value )

inline

References debug.

Referenced by main(), test_custom_serialization(), and test_custom_worldobject().

◆ set_fence()

void madness::Cloud::set_fence ( bool value )

inline

References dofence.

◆ set_force_load_from_cache()

void madness::Cloud::set_force_load_from_cache ( bool value )

inline

References force_load_from_cache.

Referenced by main(), and test_custom_worldobject().

◆ store()

template<typename T >

recordlistT madness::Cloud::store	(	madness::World &	world,
		const T &	source
	)

inline

Parameters

[in] world presumably the universe

References dofence, madness::WorldGopInterface::fence(), madness::World::gop, is_replicated, MADNESS_EXCEPTION, madness::print(), source(), store_other(), store_tuple(), and writing_time.

Referenced by chunk_example(), madness::CCPair::cloud_store(), madness::CCIntermediatePotentials::cloud_store(), madness::Info::cloud_store(), custom_serialize_tester::cloud_store(), main(), madness::MacroTask< taskT >::prepare_output_records(), simple_example(), store_tuple(), test_custom_serialization(), and test_custom_worldobject().

◆ store_other() [1/2]

template<typename T >

recordlistT madness::Cloud::store_other	(	madness::World &	world,
		const std::vector< T > &	source
	)

inline

References container, debug, dofence, madness::WorldGopInterface::fence(), madness::World::gop, madness::name(), madness::World::size(), source(), and store_other().

◆ store_other() [2/2]

template<typename T >

recordlistT madness::Cloud::store_other	(	madness::World &	world,
		const T &	source
	)

inlineprivate

References cache_stores, madness::Recordlist< keyT >::compute_record(), container, debug, dofence, madness::WorldGopInterface::fence(), madness::World::gop, is_in_container(), local_list_of_container_keys, madness::name(), madness::World::rank(), source(), T(), and writing_time1.

Referenced by store(), and store_other().

◆ store_tuple()

template<typename... Ts>

recordlistT madness::Cloud::store_tuple	(	World &	world,
		const std::tuple< Ts... > &	input
	)

inline

store a tuple in multiple records

References madness::arg(), store(), and v.

Referenced by store().

Member Data Documentation

◆ cache_reads

std::atomic<long> madness::Cloud::cache_reads =0l

mutableprivate

Referenced by clear_timings(), load_from_cache(), and print_timings().

◆ cache_stores

std::atomic<long> madness::Cloud::cache_stores =0l

mutableprivate

Referenced by clear_timings(), print_timings(), and store_other().

◆ cached_objects

cacheT madness::Cloud::cached_objects

private

Referenced by cache(), clear_cache(), is_cached(), and load_from_cache().

◆ container

madness::WorldContainer<keyT, valueT> madness::Cloud::container

private

Referenced by clear(), do_load(), print_size(), replicate(), store_other(), and store_other().

◆ debug

bool madness::Cloud::debug = false

private

prints debug output

Referenced by do_load(), load_from_cache(), load_tuple(), set_debug(), store_other(), and store_other().

◆ dofence

bool madness::Cloud::dofence = true

private

fences after load/store

Referenced by set_fence(), store(), store_other(), and store_other().

◆ force_load_from_cache

bool madness::Cloud::force_load_from_cache = false

private

forces load from cache (mainly for debugging)

Referenced by do_load(), and set_force_load_from_cache().

◆ is_replicated

bool madness::Cloud::is_replicated =false

private

if contents of the container are replicated

Referenced by print_size(), replicate(), and store().

◆ local_list_of_container_keys

recordlistT madness::Cloud::local_list_of_container_keys

private

Referenced by clear_cache(), is_in_container(), and store_other().

◆ reading_time

std::atomic<long> madness::Cloud::reading_time =0l

mutableprivate

Referenced by clear_timings(), load(), and print_timings().

◆ replication_time

std::atomic<long> madness::Cloud::replication_time =0l

mutableprivate

Referenced by clear_timings(), print_timings(), and replicate().

◆ writing_time

std::atomic<long> madness::Cloud::writing_time =0l

mutableprivate

Referenced by clear_timings(), print_timings(), and store().

◆ writing_time1

std::atomic<long> madness::Cloud::writing_time1 =0l

mutableprivate

Referenced by clear_timings(), and store_other().

The documentation for this class was generated from the following file:

cloud.h

Classes

Public Types

Public Member Functions

Private Types

Private Member Functions

Private Attributes

Detailed Description

Member Typedef Documentation

◆ cached_objT

◆ cacheT

◆ has_cloud_serialize

◆ is_parallel_serializable_object

◆ is_world_constructible

◆ keyT

◆ member_cloud_serialize_t

◆ recordlistT

◆ valueT

Constructor & Destructor Documentation

◆ Cloud()

Member Function Documentation

◆ allocator()

◆ cache()

◆ clear()

◆ clear_cache()

◆ clear_timings()

◆ do_load() [1/2]

◆ do_load() [2/2]

◆ forward_load()

◆ is_cached()

◆ is_in_container()

◆ load()

◆ load_from_cache()

◆ load_tuple()

◆ print_size()

◆ print_timings()

◆ replicate()

◆ set_debug()

◆ set_fence()

◆ set_force_load_from_cache()

◆ store()

◆ store_other() [1/2]

◆ store_other() [2/2]

◆ store_tuple()

Member Data Documentation

◆ cache_reads

◆ cache_stores

◆ cached_objects

◆ container

◆ debug

◆ dofence

◆ force_load_from_cache

◆ is_replicated

◆ local_list_of_container_keys

◆ reading_time

◆ replication_time

◆ writing_time

◆ writing_time1