MADNESS  0.10.1
worlddc.h
Go to the documentation of this file.
1 /*
2  This file is part of MADNESS.
3 
4  Copyright (C) 2007,2010 Oak Ridge National Laboratory
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10 
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  GNU General Public License for more details.
15 
16  You should have received a copy of the GNU General Public License
17  along with this program; if not, write to the Free Software
18  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 
20  For more information please contact:
21 
22  Robert J. Harrison
23  Oak Ridge National Laboratory
24  One Bethel Valley Road
25  P.O. Box 2008, MS-6367
26 
27  email: harrisonrj@ornl.gov
28  tel: 865-241-3937
29  fax: 865-572-0680
30 */
31 
32 #ifndef MADNESS_WORLD_WORLDDC_H__INCLUDED
33 #define MADNESS_WORLD_WORLDDC_H__INCLUDED
34 
35 /*!
36  \file worlddc.h
37  \brief Implements WorldContainer
38  \addtogroup worlddc
39  @{
40 
41 */
42 
43 #include <functional>
44 #include <set>
45 
50 
51 namespace madness
52 {
53 
54  template <typename keyT, typename valueT, typename hashfunT>
55  class WorldContainer;
56 
57  template <typename keyT, typename valueT, typename hashfunT>
58  class WorldContainerImpl;
59 
60  template <typename keyT, typename valueT, typename hashfunT>
61  void swap(WorldContainer<keyT, valueT, hashfunT> &, WorldContainer<keyT, valueT, hashfunT> &);
62 
63  template <typename keyT>
64  class WorldDCPmapInterface;
65 
66  template <typename keyT>
68  {
69  public:
70  virtual std::size_t size() const = 0;
71  virtual void redistribute_phase1(const std::shared_ptr<WorldDCPmapInterface<keyT>> &newmap) = 0;
72  virtual void redistribute_phase2() = 0;
73  virtual void redistribute_phase3() = 0;
75  };
76 
77  /// Interface to be provided by any process map
78 
79  /// \ingroup worlddc
80  template <typename keyT>
82  {
83  public:
85 
86  private:
87  std::set<ptrT> ptrs;
88 
89  public:
90  /// Maps key to processor
91 
92  /// @param[in] key Key for container
93  /// @return Processor that logically owns the key
94  virtual ProcessID owner(const keyT &key) const = 0;
95 
96  virtual ~WorldDCPmapInterface() {}
97 
98  virtual void print() const {}
99 
100  /// Registers object for receipt of redistribute callbacks
101 
102  /// @param[in] ptr Pointer to class derived from WorldDCRedistributedInterface
104  {
105  ptrs.insert(ptr);
106  }
107 
108  /// Deregisters object for receipt of redistribute callbacks
109 
110  /// @param[in] ptr Pointer to class derived from WorldDCRedistributedInterface
112  {
113  ptrs.erase(ptr);
114  }
115 
116  /// Invoking this switches all registered objects from this process map to the new one
117 
118  /// After invoking this routine all objects will be registered with the
119  /// new map and no objects will be registered in the current map.
120  /// @param[in] world The associated world
121  /// @param[in] newpmap The new process map
122  void redistribute(World &world, const std::shared_ptr<WorldDCPmapInterface<keyT>> &newpmap)
123  {
124  print_data_sizes(world, "before redistributing");
125  world.gop.fence();
126  for (typename std::set<ptrT>::iterator iter = ptrs.begin();
127  iter != ptrs.end();
128  ++iter)
129  {
130  (*iter)->redistribute_phase1(newpmap);
131  }
132  world.gop.fence();
133  for (typename std::set<ptrT>::iterator iter = ptrs.begin();
134  iter != ptrs.end();
135  ++iter)
136  {
137  (*iter)->redistribute_phase2();
138  newpmap->register_callback(*iter);
139  }
140  world.gop.fence();
141  for (typename std::set<ptrT>::iterator iter = ptrs.begin();
142  iter != ptrs.end();
143  ++iter)
144  {
145  (*iter)->redistribute_phase3();
146  }
147  world.gop.fence();
148  ptrs.clear();
149  newpmap->print_data_sizes(world, "after redistributing");
150  }
151 
152  /// Counts global number of entries in all containers associated with this process map
153 
154  /// Collective operation with global fence
155  std::size_t global_size(World &world) const
156  {
157  world.gop.fence();
158  std::size_t sum = local_size();
159  world.gop.sum(sum);
160  world.gop.fence();
161  return sum;
162  }
163 
164  /// Counts local number of entries in all containers associated with this process map
165  std::size_t local_size() const
166  {
167  std::size_t sum = 0;
168  for (typename std::set<ptrT>::iterator iter = ptrs.begin(); iter != ptrs.end(); ++iter)
169  {
170  sum += (*iter)->size();
171  }
172  return sum;
173  }
174 
175  /// Prints size info to std::cout
176 
177  /// Collective operation with global fence
178  void print_data_sizes(World &world, const std::string msg = "") const
179  {
180  world.gop.fence();
181  std::size_t total = global_size(world);
182  std::vector<std::size_t> sizes(world.size());
183  sizes[world.rank()] = local_size();
184  world.gop.sum(&sizes[0], world.size());
185  if (world.rank() == 0)
186  {
187  madness::print("data distribution info", msg);
188  madness::print(" total: ", total);
189  std::cout << " procs: ";
190  for (int i = 0; i < world.size(); i++)
191  std::cout << sizes[i] << " ";
192  std::cout << std::endl;
193  }
194  world.gop.fence();
195  }
196  };
197 
198  /// Default process map is "random" using madness::hash(key)
199 
200  /// \ingroup worlddc
201  template <typename keyT, typename hashfunT = Hash<keyT>>
203  {
204  private:
205  const int nproc;
206  hashfunT hashfun;
207 
208  public:
209  WorldDCDefaultPmap(World &world, const hashfunT &hf = hashfunT()) : nproc(world.mpi.nproc()),
210  hashfun(hf)
211  {
212  }
213 
214  ProcessID owner(const keyT &key) const
215  {
216  if (nproc == 1)
217  return 0;
218  return hashfun(key) % nproc;
219  }
220  };
221 
222  /// Local process map will always return the current process as owner
223 
224  /// \ingroup worlddc
225  template <typename keyT, typename hashfunT = Hash<keyT>>
227  {
228  private:
230 
231  public:
232  WorldDCLocalPmap(World &world) : me(world.rank()) {}
233  ProcessID owner(const keyT &key) const
234  {
235  return me;
236  }
237  };
238 
239  /// Iterator for distributed container wraps the local iterator
240 
241  /// \ingroup worlddc
242  template <class internal_iteratorT>
244  {
245  public:
246  typedef typename std::iterator_traits<internal_iteratorT>::iterator_category iterator_category;
247  typedef typename std::iterator_traits<internal_iteratorT>::value_type value_type;
248  typedef typename std::iterator_traits<internal_iteratorT>::difference_type difference_type;
249  typedef typename std::iterator_traits<internal_iteratorT>::pointer pointer;
250  typedef typename std::iterator_traits<internal_iteratorT>::reference reference;
251 
252  private:
253  internal_iteratorT it; ///< Iterator from local container
254  // TODO: Convert this to a scoped pointer.
255  mutable value_type *value; ///< holds the remote values
256 
257  public:
258  /// Default constructor makes a local uninitialized value
260  : it(), value(nullptr) {}
261 
262  /// Initializes from a local iterator
263  explicit WorldContainerIterator(const internal_iteratorT &it)
264  : it(it), value(nullptr) {}
265 
266  /// Initializes to cache a remote value
268  : it(), value(nullptr)
269  {
270  value = new value_type(v);
271  }
272 
274  : it(), value(nullptr)
275  {
276  copy(other);
277  }
278 
279  template <class iteratorT>
281  : it(), value(nullptr)
282  {
283  copy(other);
284  }
285 
287  {
288  delete value;
289  }
290 
291  /// Assignment
293  {
294  copy(other);
295  return *this;
296  }
297 
298  /// Determines if two iterators are identical
299  bool operator==(const WorldContainerIterator &other) const
300  {
301  return (((!is_cached()) && (!other.is_cached())) && it == other.it) ||
302  ((is_cached() && other.is_cached()) && value->first == other.value->first);
303  }
304 
305  /// Determines if two iterators are different
306  bool operator!=(const WorldContainerIterator &other) const
307  {
308  return !(*this == other);
309  }
310 
311  /// Pre-increment of an iterator (i.e., ++it) --- \em local iterators only
312 
313  /// Trying to increment a remote iterator will throw
315  {
317  ++it;
318  return *this;
319  }
320 
322  {
325  ++it;
326  return result;
327  }
328 
329  /// Iterators dereference to std::pair<const keyT,valueT>
331  {
332  return (is_cached() ? value : it.operator->());
333  }
334 
335  /// Iterators dereference to std::pair<const keyT,valueT>
337  {
338  return (is_cached() ? *value : *it);
339  }
340 
341  /// Private: (or should be) Returns iterator of internal container
342  const internal_iteratorT &get_internal_iterator() const
343  {
344  return it;
345  }
346 
347  /// Returns true if this is non-local or cached value
348  bool is_cached() const
349  {
350  return value != nullptr;
351  }
352 
353  template <typename Archive>
354  void serialize(const Archive &)
355  {
356  MADNESS_EXCEPTION("Serializing DC iterator ... why?", false);
357  }
358 
359  private:
360  template <class iteratorT>
362 
363  template <class iteratorT>
365  {
366  if (static_cast<const void *>(this) != static_cast<const void *>(&other))
367  {
368  delete value;
369  if (other.is_cached())
370  {
371  value = new value_type(*other.value);
372  it = internal_iteratorT();
373  }
374  else
375  {
376  it = other.it;
377  value = nullptr;
378  }
379  }
380  }
381  };
382 
383  /// Internal implementation of distributed container to facilitate shallow copy
384 
385  /// \ingroup worlddc
386  template <typename keyT, typename valueT, typename hashfunT>
388  : public WorldObject<WorldContainerImpl<keyT, valueT, hashfunT>>,
389  public WorldDCRedistributeInterface<keyT>
390 #ifndef MADNESS_DISABLE_SHARED_FROM_THIS
391  ,
392  public std::enable_shared_from_this<WorldContainerImpl<keyT, valueT, hashfunT>>
393 #endif // MADNESS_DISABLE_SHARED_FROM_THIS
394  {
395  public:
396  typedef typename std::pair<const keyT, valueT> pairT;
397  typedef const pairT const_pairT;
399 
401 
402  // typedef WorldObject< WorldContainerImpl<keyT, valueT, hashfunT> > worldobjT;
403 
412 
413  friend class WorldContainer<keyT, valueT, hashfunT>;
414 
415  // template <typename containerT, typename datumT>
416  // inline
417  // static
418  // typename containerT::iterator replace(containerT& c, const datumT& d) {
419  // std::pair<typename containerT::iterator,bool> p = c.insert(d);
420  // if (!p.second) p.first->second = d.second; // Who's on first?
421  // return p.first;
422  // }
423 
424  private:
425  WorldContainerImpl(); // Inhibit default constructor
426 
427  std::shared_ptr<WorldDCPmapInterface<keyT>> pmap; ///< Function/class to map from keys to owning process
428  const ProcessID me; ///< My MPI rank
429  internal_containerT local; ///< Locally owned data
430  std::vector<keyT> *move_list; ///< Tempoary used to record data that needs redistributing
431 
432  /// Handles find request
433  void find_handler(ProcessID requestor, const keyT &key, const RemoteReference<FutureImpl<iterator>> &ref)
434  {
435  internal_iteratorT r = local.find(key);
436  if (r == local.end())
437  {
438  // print("find_handler: failure:", key);
439  this->send(requestor, &implT::find_failure_handler, ref);
440  }
441  else
442  {
443  // print("find_handler: success:", key, r->first, r->second);
444  this->send(requestor, &implT::find_success_handler, ref, *r);
445  }
446  }
447 
448  /// Handles successful find response
450  {
451  FutureImpl<iterator> *f = ref.get();
452  f->set(iterator(datum));
453  // print("find_success_handler: success:", datum.first, datum.second, f->get()->first, f->get()->second);
454  // Todo: Look at this again.
455  // ref.reset(); // Matching inc() in find() where ref was made
456  }
457 
458  /// Handles unsuccessful find response
460  {
461  FutureImpl<iterator> *f = ref.get();
462  f->set(end());
463  // print("find_failure_handler");
464  // Todo: Look at this again.
465  // ref.reset(); // Matching inc() in find() where ref was made
466  }
467 
468  public:
470  const std::shared_ptr<WorldDCPmapInterface<keyT>> &pm,
471  const hashfunT &hf)
472  : WorldObject<WorldContainerImpl<keyT, valueT, hashfunT>>(world), pmap(pm), me(world.mpi.rank()), local(5011, hf)
473  {
474  pmap->register_callback(this);
475  }
476 
478  {
479  pmap->deregister_callback(this);
480  }
481 
482  const std::shared_ptr<WorldDCPmapInterface<keyT>> &get_pmap() const
483  {
484  return pmap;
485  }
486 
487  std::shared_ptr<WorldDCPmapInterface<keyT>> &get_pmap()
488  {
489  return pmap;
490  }
491 
493  {
494  pmap->deregister_callback(this);
495  pmap.reset(new WorldDCLocalPmap<keyT>(this->get_world()));
496  pmap->register_callback(this);
497  }
498 
499  /// replicates this WorldContainer on all ProcessIDs and generates a
500  /// ProcessMap where all nodes are local
501  void replicate(bool fence)
502  {
503 
504  World &world = this->get_world();
505  pmap->deregister_callback(this);
506  pmap.reset(new WorldDCLocalPmap<keyT>(world));
507  pmap->register_callback(this);
508 
509  for (ProcessID rank = 0; rank < world.size(); rank++)
510  {
511  if (rank == world.rank())
512  {
513  std::size_t sz = size();
514  world.gop.broadcast_serializable(sz, rank);
515 
516  for (auto it = begin(); it != end(); ++it)
517  {
518  keyT key = it->first;
519  valueT value = it->second;
520  world.gop.broadcast_serializable(key, rank);
521  world.gop.broadcast_serializable(value, rank);
522  }
523  }
524  else
525  {
526  size_t sz;
527  world.gop.broadcast_serializable(sz, rank);
528  for (size_t i = 0; i < sz; i++)
529  {
530  keyT key;
531  valueT value;
532  world.gop.broadcast_serializable(key, rank);
533  world.gop.broadcast_serializable(value, rank);
534  insert(pairT(key, value));
535  }
536  }
537  }
538  if (fence)
539  world.gop.fence();
540  }
541 
542  hashfunT &get_hash() const { return local.get_hash(); }
543 
544  bool is_local(const keyT &key) const
545  {
546  return owner(key) == me;
547  }
548 
549  ProcessID owner(const keyT &key) const
550  {
551  return pmap->owner(key);
552  }
553 
554  bool probe(const keyT &key) const
555  {
556  ProcessID dest = owner(key);
557  if (dest == me)
558  return local.find(key) != local.end();
559  else
560  return false;
561  }
562 
563  std::size_t size() const
564  {
565  return local.size();
566  }
567 
568  void insert(const pairT &datum)
569  {
570  ProcessID dest = owner(datum.first);
571  if (dest == me)
572  {
573  // Was using iterator ... try accessor ?????
574  accessor acc;
575  // N.B. key might already exist if want to simply replace
576  [[maybe_unused]] auto inserted = local.insert(acc, datum.first);
577  acc->second = datum.second;
578  }
579  else
580  {
581  // Must be send (not task) for sequential consistency (and relies on single-threaded remote server)
582  this->send(dest, &implT::insert, datum);
583  }
584  }
585 
586  bool insert_acc(accessor &acc, const keyT &key)
587  {
588  MADNESS_ASSERT(owner(key) == me);
589  return local.insert(acc, key);
590  }
591 
592  bool insert_const_acc(const_accessor &acc, const keyT &key)
593  {
594  MADNESS_ASSERT(owner(key) == me);
595  return local.insert(acc, key);
596  }
597 
598  void clear()
599  {
600  local.clear();
601  }
602 
603  void erase(const keyT &key)
604  {
605  ProcessID dest = owner(key);
606  if (dest == me)
607  {
608  [[maybe_unused]] auto erased = local.try_erase(key);
609  MADNESS_ASSERT(erased);
610  }
611  else
612  {
613  void (implT::*eraser)(const keyT &) = &implT::erase;
614  this->send(dest, eraser, key);
615  }
616  }
617 
618  template <typename InIter>
619  void erase(InIter it)
620  {
621  MADNESS_ASSERT(!it.is_cached());
622  MADNESS_ASSERT(it != end());
623  erase(it->first);
624  }
625 
626  template <typename InIter>
627  void erase(InIter first, InIter last)
628  {
629  InIter it = first;
630  do
631  {
632  first++;
633  erase(it->first);
634  it = first;
635  } while (first != last);
636  }
637 
639  {
640  return iterator(local.begin());
641  }
642 
644  {
645  return const_iterator(local.begin());
646  }
647 
649  {
650  return iterator(local.end());
651  }
652 
654  {
655  return const_iterator(local.end());
656  }
657 
658  Future<const_iterator> find(const keyT &key) const
659  {
660  // Ugliness here to avoid replicating find() and
661  // associated handlers for const. Assumption is that
662  // const and non-const iterators are identical except for
663  // const attribute ... at some point probably need to do
664  // the right thing.
665  Future<iterator> r = const_cast<implT *>(this)->find(key);
666  return *(Future<const_iterator> *)(&r);
667  }
668 
669  Future<iterator> find(const keyT &key)
670  {
671  ProcessID dest = owner(key);
672  if (dest == me)
673  {
674  return Future<iterator>(iterator(local.find(key)));
675  }
676  else
677  {
678  Future<iterator> result;
679  this->send(dest, &implT::find_handler, me, key, result.remote_ref(this->get_world()));
680  return result;
681  }
682  }
683 
684  bool find(accessor &acc, const keyT &key)
685  {
686  if (owner(key) != me)
687  return false;
688  return local.find(acc, key);
689  }
690 
691  bool find(const_accessor &acc, const keyT &key) const
692  {
693  if (owner(key) != me)
694  return false;
695  return local.find(acc, key);
696  }
697 
698  // Used to forward call to item member function
699  template <typename memfunT>
700  MEMFUN_RETURNT(memfunT)
701  itemfun(const keyT &key, memfunT memfun)
702  {
703  accessor acc;
704  // N.B. key may already exist, this is just to ensure lock is held by acc
705  [[maybe_unused]] auto inserted = local.insert(acc, key);
706  return (acc->second.*memfun)();
707  }
708 
709  // Used to forward call to item member function
710  template <typename memfunT, typename arg1T>
711  MEMFUN_RETURNT(memfunT)
712  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1)
713  {
714  accessor acc;
715  // N.B. key may already exist, this is just to ensure lock is held by acc
716  [[maybe_unused]] auto inserted = local.insert(acc, key);
717  return (acc->second.*memfun)(arg1);
718  }
719 
720  // Used to forward call to item member function
721  template <typename memfunT, typename arg1T, typename arg2T>
722  MEMFUN_RETURNT(memfunT)
723  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2)
724  {
725  accessor acc;
726  // N.B. key may already exist, this is just to ensure lock is held by acc
727  [[maybe_unused]] auto inserted = local.insert(acc, key);
728  return (acc->second.*memfun)(arg1, arg2);
729  }
730 
731  // Used to forward call to item member function
732  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T>
733  MEMFUN_RETURNT(memfunT)
734  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3)
735  {
736  accessor acc;
737  // N.B. key may already exist, this is just to ensure lock is held by acc
738  [[maybe_unused]] auto inserted = local.insert(acc, key);
739  return (acc->second.*memfun)(arg1, arg2, arg3);
740  }
741 
742  // Used to forward call to item member function
743  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T>
744  MEMFUN_RETURNT(memfunT)
745  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4)
746  {
747  accessor acc;
748  // N.B. key may already exist, this is just to ensure lock is held by acc
749  [[maybe_unused]] auto inserted = local.insert(acc, key);
750  return (acc->second.*memfun)(arg1, arg2, arg3, arg4);
751  }
752 
753  // Used to forward call to item member function
754  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T>
755  MEMFUN_RETURNT(memfunT)
756  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5)
757  {
758  accessor acc;
759  // N.B. key may already exist, this is just to ensure lock is held by acc
760  [[maybe_unused]] auto inserted = local.insert(acc, key);
761  return (acc->second.*memfun)(arg1, arg2, arg3, arg4, arg5);
762  }
763 
764  // Used to forward call to item member function
765  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T>
766  MEMFUN_RETURNT(memfunT)
767  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6)
768  {
769  accessor acc;
770  // N.B. key may already exist, this is just to ensure lock is held by acc
771  [[maybe_unused]] auto inserted = local.insert(acc, key);
772  return (acc->second.*memfun)(arg1, arg2, arg3, arg4, arg5, arg6);
773  }
774 
775  // Used to forward call to item member function
776  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T, typename arg7T>
777  MEMFUN_RETURNT(memfunT)
778  itemfun(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3,
779  const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7)
780  {
781  accessor acc;
782  // N.B. key may already exist, this is just to ensure lock is held by acc
783  [[maybe_unused]] auto inserted = local.insert(acc, key);
784  return (acc->second.*memfun)(arg1, arg2, arg3, arg4, arg5, arg6, arg7);
785  }
786 
787  // First phase of redistributions changes pmap and makes list of stuff to move
788  void redistribute_phase1(const std::shared_ptr<WorldDCPmapInterface<keyT>> &newpmap)
789  {
790  pmap = newpmap;
791  move_list = new std::vector<keyT>();
792  for (typename internal_containerT::iterator iter = local.begin(); iter != local.end(); ++iter)
793  {
794  if (owner(iter->first) != me)
795  move_list->push_back(iter->first);
796  }
797  }
798 
799  struct P2Op
800  {
804  P2Op(const P2Op &p) : impl(p.impl) {}
805  bool operator()(typename rangeT::iterator &iterator) const
806  {
808  MADNESS_ASSERT(iter != impl->local.end());
809 
810  // impl->insert(*iter);
811  impl->task(impl->owner(*iterator), &implT::insert, *iter);
812 
813  impl->local.erase(iter); // delete local copy of the data
814  return true;
815  }
816  };
817 
818  // Second phase moves data
820  {
821  this->get_world().taskq.for_each(typename P2Op::rangeT(move_list->begin(), move_list->end()), P2Op(this));
822  // std::vector<keyT>& mvlist = *move_list;
823  // for (unsigned int i=0; i<move_list->size(); ++i) {
824  // typename internal_containerT::iterator iter = local.find(mvlist[i]);
825  // MADNESS_ASSERT(iter != local.end());
826  // insert(*iter);
827  // local.erase(iter);
828  // }
829  // delete move_list;
830  }
831 
832  // Third phase cleans up
834  {
835  delete move_list;
836  }
837  };
838 
839  /// Makes a distributed container with specified attributes
840 
841  /// \ingroup worlddc
842  ///
843  /// There is no communication or syncronization associated with
844  /// making a new container, but every process must invoke the
845  /// constructor for each container in the same order. This is so
846  /// that we can assign each container a unique ID without any
847  /// communication. Since remotely invoked operations may start
848  /// happening before local construction, messages on not yet
849  /// constructed containers are buffered pending construction.
850  ///
851  /// Similarly, when a container is destroyed, the actual
852  /// destruction is deferred until a synchronization point
853  /// (world.gop.fence()) in order to eliminate the need to fence
854  /// before destroying every container.
855  ///
856  /// The distribution of data between processes is controlled by
857  /// the process map (Pmap) class. The default is uniform
858  /// hashing based upon a strong (Bob Jenkins, lookup3) bytewise
859  /// hash of the key.
860  ///
861  /// All operations, including constructors and destructors, are
862  /// non-blocking and return immediately. If communication occurs
863  /// it is asynchronous, otherwise operations are local.
864  template <typename keyT, typename valueT, typename hashfunT = Hash<keyT>>
866  {
867  public:
870  typedef typename implT::pairT pairT;
871  typedef typename implT::iterator iterator;
873  typedef typename implT::accessor accessor;
877 
878  private:
879  std::shared_ptr<implT> p;
880 
881  inline void check_initialized() const
882  {
883  MADNESS_ASSERT(p);
884  }
885 
886  public:
887  /// Makes an uninitialized container (no communication)
888 
889  /// The container is useless until assigned to from a fully
890  /// constructed container. There is no need to worry about
891  /// default constructors being executed in order.
893  : p()
894  {
895  }
896 
897  /// Makes an initialized, empty container with default data distribution (no communication)
898 
899  /// A unique ID is associated with every distributed container
900  /// within a world. In order to avoid synchronization when
901  /// making a container, we have to assume that all processes
902  /// execute this constructor in the same order (does not apply
903  /// to the non-initializing, default constructor).
904  WorldContainer(World &world, bool do_pending = true, const hashfunT &hf = hashfunT())
905  : p(new implT(world,
906  std::shared_ptr<WorldDCPmapInterface<keyT>>(new WorldDCDefaultPmap<keyT, hashfunT>(world, hf)),
907  hf))
908  {
909  if (do_pending)
910  p->process_pending();
911  }
912 
913  /// Makes an initialized, empty container (no communication)
914 
915  /// A unique ID is associated with every distributed container
916  /// within a world. In order to avoid synchronization when
917  /// making a container, we have to assume that all processes
918  /// execute this constructor in the same order (does not apply
919  /// to the non-initializing, default constructor).
921  const std::shared_ptr<WorldDCPmapInterface<keyT>> &pmap,
922  bool do_pending = true,
923  const hashfunT &hf = hashfunT())
924  : p(new implT(world, pmap, hf))
925  {
926  if (do_pending)
927  p->process_pending();
928  }
929 
930  /// Copy constructor is shallow (no communication)
931 
932  /// The copy refers to exactly the same container as other
933  /// which must be initialized.
935  : p(other.p)
936  {
938  }
939 
940  /// Assignment is shallow (no communication)
941 
942  /// The copy refers to exactly the same container as other
943  /// which must be initialized.
945  {
946  if (this != &other)
947  {
948  other.check_initialized();
949  p = other.p;
950  }
951  return *this;
952  }
953 
954  /// Returns the world associated with this container
955  World &get_world() const
956  {
958  return p->get_world();
959  }
960 
961  std::shared_ptr<WorldDCPmapInterface<keyT>> &get_impl()
962  {
964  return p;
965  }
966 
967  /// replicates this WorldContainer on all ProcessIDs
968  void replicate(bool fence = true)
969  {
970  p->replicate(fence);
971  }
972 
973  /// Inserts/replaces key+value pair (non-blocking communication if key not local)
974  void replace(const pairT &datum)
975  {
977  p->insert(datum);
978  }
979 
980  /// Inserts/replaces key+value pair (non-blocking communication if key not local)
981  void replace(const keyT &key, const valueT &value)
982  {
983  replace(pairT(key, value));
984  }
985 
986  /// Write access to LOCAL value by key. Returns true if found, false otherwise (always false for remote).
987  bool find(accessor &acc, const keyT &key)
988  {
990  return p->find(acc, key);
991  }
992 
993  /// Read access to LOCAL value by key. Returns true if found, false otherwise (always false for remote).
994  bool find(const_accessor &acc, const keyT &key) const
995  {
997  return p->find(acc, key);
998  }
999 
1000  /// Write access to LOCAL value by key. Returns true if inserted, false if already exists (throws if remote)
1001  bool insert(accessor &acc, const keyT &key)
1002  {
1004  return p->insert_acc(acc, key);
1005  }
1006 
1007  /// Read access to LOCAL value by key. Returns true if inserted, false if already exists (throws if remote)
1008  bool insert(const_accessor &acc, const keyT &key)
1009  {
1011  return p->insert_acc(acc, key);
1012  }
1013 
1014  /// Inserts pairs (non-blocking communication if key(s) not local)
1015  template <typename input_iterator>
1016  void replace(input_iterator &start, input_iterator &end)
1017  {
1019  using std::placeholders::_1;
1020  std::for_each(start, end, std::bind(this, std::mem_fn(&containerT::insert), _1));
1021  }
1022 
1023  /// Returns true if local data is immediately available (no communication)
1024  bool probe(const keyT &key) const
1025  {
1027  return p->probe(key);
1028  }
1029 
1030  /// Returns processor that logically owns key (no communication)
1031 
1032  /// Local remapping may have changed its physical location, but all
1033  /// operations should forward correctly.
1034  inline ProcessID owner(const keyT &key) const
1035  {
1037  return p->owner(key);
1038  }
1039 
1040  /// Returns true if the key maps to the local processor (no communication)
1041  bool is_local(const keyT &key) const
1042  {
1044  return p->is_local(key);
1045  }
1046 
1047  /// Returns a future iterator (non-blocking communication if key not local)
1048 
1049  /// Like an std::map an iterator "points" to an std::pair<const keyT,valueT>.
1050  ///
1051  /// Refer to Future for info on how to avoid blocking.
1052  Future<iterator> find(const keyT &key)
1053  { //
1055  return p->find(key);
1056  }
1057 
1058  /// Returns a future iterator (non-blocking communication if key not local)
1059 
1060  /// Like an std::map an iterator "points" to an std::pair<const keyT,valueT>.
1061  ///
1062  /// Refer to Future for info on how to avoid blocking.
1063  Future<const_iterator> find(const keyT &key) const
1064  {
1066  return const_cast<const implT *>(p.get())->find(key);
1067  }
1068 
1069  /// Returns an iterator to the beginning of the \em local data (no communication)
1071  {
1073  return p->begin();
1074  }
1075 
1076  /// Returns an iterator to the beginning of the \em local data (no communication)
1078  {
1080  return const_cast<const implT *>(p.get())->begin();
1081  }
1082 
1083  /// Returns an iterator past the end of the \em local data (no communication)
1085  {
1087  return p->end();
1088  }
1089 
1090  /// Returns an iterator past the end of the \em local data (no communication)
1092  {
1094  return const_cast<const implT *>(p.get())->end();
1095  }
1096 
1097  /// Erases entry from container (non-blocking comm if remote)
1098 
1099  /// Missing keys are quietly ignored.
1100  ///
1101  /// Note that erasing an entry may invalidate iterators on the
1102  /// remote end. This is just the same as what happens when
1103  /// using STL iterators on an STL container in a sequential
1104  /// algorithm.
1105  void erase(const keyT &key)
1106  {
1108  p->erase(key);
1109  }
1110 
1111  /// Erases entry corresponding to \em local iterator (no communication)
1112  void erase(const iterator &it)
1113  {
1115  p->erase(it);
1116  }
1117 
1118  /// Erases range defined by \em local iterators (no communication)
1119  void erase(const iterator &start, const iterator &finish)
1120  {
1122  p->erase(start, finish);
1123  }
1124 
1125  /// Clears all \em local data (no communication)
1126 
1127  /// Invalidates all iterators
1128  void clear()
1129  {
1131  p->clear();
1132  }
1133 
1134  /// Returns the number of \em local entries (no communication)
1135  std::size_t size() const
1136  {
1138  return p->size();
1139  }
1140 
1141  /// Returns shared pointer to the process mapping
1142  inline const std::shared_ptr<WorldDCPmapInterface<keyT>> &get_pmap() const
1143  {
1145  return p->get_pmap();
1146  }
1147 
1148  /// Returns shared pointer to the process mapping
1149  inline void reset_pmap_to_local()
1150  {
1151  p->reset_pmap_to_local();
1152  }
1153 
1154  /// Returns a reference to the hashing functor
1155  hashfunT &get_hash() const
1156  {
1158  return p->get_hash();
1159  }
1160 
1161  /// Process pending messages
1162 
1163  /// If the constructor was given \c do_pending=false then you
1164  /// \em must invoke this routine in order to process both
1165  /// prior and future messages.
1166  inline void process_pending()
1167  {
1169  p->process_pending();
1170  }
1171 
1172  /// Sends message "resultT memfun()" to item (non-blocking comm if remote)
1173 
1174  /// If item does not exist it is made with the default constructor.
1175  ///
1176  /// Future arguments must be ready for remote messages.
1177  ///
1178  /// Returns a future result (Future<void> may be ignored).
1179  ///
1180  /// The method executes with a write lock on the item.
1181  template <typename memfunT>
1182  Future<MEMFUN_RETURNT(memfunT)>
1183  send(const keyT &key, memfunT memfun)
1184  {
1186  MEMFUN_RETURNT(memfunT)
1187  (implT::*itemfun)(const keyT &, memfunT) = &implT::template itemfun<memfunT>;
1188  return p->send(owner(key), itemfun, key, memfun);
1189  }
1190 
1191  /// Sends message "resultT memfun(arg1T)" to item (non-blocking comm if remote)
1192 
1193  /// If item does not exist it is made with the default constructor.
1194  ///
1195  /// Future arguments must be ready for remote messages.
1196  ///
1197  /// Returns a future result (Future<void> may be ignored).
1198  ///
1199  /// The method executes with a write lock on the item.
1200  template <typename memfunT, typename arg1T>
1201  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1202  send(const keyT &key, const memfunT &memfun, const arg1T &arg1)
1203  {
1205  // To work around bug in g++ 4.3.* use static cast as alternative mechanism to force type deduction
1206  MEMFUN_RETURNT(memfunT)
1207  (implT::*itemfun)(const keyT &, memfunT, const arg1T &) = &implT::template itemfun<memfunT, arg1T>;
1208  return p->send(owner(key), itemfun, key, memfun, arg1);
1209  /*return p->send(owner(key),
1210  static_cast<MEMFUN_RETURNT(memfunT)(implT::*)(const keyT&, memfunT, const arg1T&)>(&implT:: template itemfun<memfunT,arg1T>),
1211  key, memfun, arg1);*/
1212  }
1213 
1214  /// Sends message "resultT memfun(arg1T,arg2T)" to item (non-blocking comm if remote)
1215 
1216  /// If item does not exist it is made with the default constructor.
1217  ///
1218  /// Future arguments must be ready for both local and remote messages.
1219  ///
1220  /// Returns a future result (Future<void> may be ignored).
1221  ///
1222  /// The method executes with a write lock on the item.
1223  template <typename memfunT, typename arg1T, typename arg2T>
1224  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1225  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2)
1226  {
1228  // To work around bug in g++ 4.3.* use static cast as alternative mechanism to force type deduction
1229  MEMFUN_RETURNT(memfunT)
1230  (implT::*itemfun)(const keyT &, memfunT, const arg1T &, const arg2T &) = &implT::template itemfun<memfunT, arg1T, arg2T>;
1231  return p->send(owner(key), itemfun, key, memfun, arg1, arg2);
1232  /*return p->send(owner(key),
1233  static_cast<MEMFUN_RETURNT(memfunT)(implT::*)(const keyT&, memfunT, const arg1T&, const arg2T&)>(&implT:: template itemfun<memfunT,arg1T,arg2T>), key, memfun, arg1, arg2);*/
1234  }
1235 
1236  /// Sends message "resultT memfun(arg1T,arg2T,arg3T)" to item (non-blocking comm if remote)
1237 
1238  /// If item does not exist it is made with the default constructor.
1239  ///
1240  /// Future arguments must be ready for both local and remote messages.
1241  ///
1242  /// Returns a future result (Future<void> may be ignored).
1243  ///
1244  /// The method executes with a write lock on the item.
1245  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T>
1246  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1247  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3)
1248  {
1250  MEMFUN_RETURNT(memfunT)
1251  (implT::*itemfun)(const keyT &, memfunT, const arg1T &, const arg2T &, const arg3T &) = &implT::template itemfun<memfunT, arg1T, arg2T, arg3T>;
1252  return p->send(owner(key), itemfun, key, memfun, arg1, arg2, arg3);
1253  }
1254 
1255  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T)" to item (non-blocking comm if remote)
1256 
1257  /// If item does not exist it is made with the default constructor.
1258  ///
1259  /// Future arguments must be ready for both local and remote messages.
1260  ///
1261  /// Returns a future result (Future<void> may be ignored).
1262  ///
1263  /// The method executes with a write lock on the item.
1264  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T>
1265  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1266  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4)
1267  {
1269  MEMFUN_RETURNT(memfunT)
1270  (implT::*itemfun)(const keyT &, memfunT, const arg1T &, const arg2T &, const arg3T &, const arg4T &) = &implT::template itemfun<memfunT, arg1T, arg2T, arg3T, arg4T>;
1271  return p->send(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4);
1272  }
1273 
1274  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T)" to item (non-blocking comm if remote)
1275 
1276  /// If item does not exist it is made with the default constructor.
1277  ///
1278  /// Future arguments must be ready for both local and remote messages.
1279  ///
1280  /// Returns a future result (Future<void> may be ignored).
1281  ///
1282  /// The method executes with a write lock on the item.
1283  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T>
1284  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1285  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5)
1286  {
1288  MEMFUN_RETURNT(memfunT)
1289  (implT::*itemfun)(const keyT &, memfunT, const arg1T &, const arg2T &, const arg3T &, const arg4T &, const arg5T &) = &implT::template itemfun<memfunT, arg1T, arg2T, arg3T, arg4T, arg5T>;
1290  return p->send(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, arg5);
1291  }
1292 
1293  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T)" to item (non-blocking comm if remote)
1294 
1295  /// If item does not exist it is made with the default constructor.
1296  ///
1297  /// Future arguments must be ready for both local and remote messages.
1298  ///
1299  /// Returns a future result (Future<void> may be ignored).
1300  ///
1301  /// The method executes with a write lock on the item.
1302  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T>
1303  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1304  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6)
1305  {
1307  MEMFUN_RETURNT(memfunT)
1308  (implT::*itemfun)(const keyT &, memfunT, const arg1T &, const arg2T &, const arg3T &, const arg4T &, const arg5T &, const arg6T &) = &implT::template itemfun<memfunT, arg1T, arg2T, arg3T, arg4T, arg5T, arg6T>;
1309  return p->send(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, arg5, arg6);
1310  }
1311 
1312  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T)" to item (non-blocking comm if remote)
1313 
1314  /// If item does not exist it is made with the default constructor.
1315  ///
1316  /// Future arguments must be ready for both local and remote messages.
1317  ///
1318  /// Returns a future result (Future<void> may be ignored).
1319  ///
1320  /// The method executes with a write lock on the item.
1321  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T, typename arg7T>
1322  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1323  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4,
1324  const arg5T &arg5, const arg6T &arg6, const arg7T &arg7)
1325  {
1327  MEMFUN_RETURNT(memfunT)
1328  (implT::*itemfun)(const keyT &, memfunT, const arg1T &, const arg2T &, const arg3T &, const arg4T &, const arg5T &, const arg6T &, const arg7T &) = &implT::template itemfun<memfunT, arg1T, arg2T, arg3T, arg4T, arg5T, arg6T, arg7T>;
1329  return p->send(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
1330  }
1331 
1332  /// Sends message "resultT memfun() const" to item (non-blocking comm if remote)
1333 
1334  /// The method executes with a write lock on the item.
1335  template <typename memfunT>
1336  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1337  send(const keyT &key, memfunT memfun) const
1338  {
1339  return const_cast<containerT *>(this)->send(key, memfun);
1340  }
1341 
1342  /// Sends message "resultT memfun(arg1T) const" to item (non-blocking comm if remote)
1343 
1344  /// The method executes with a write lock on the item.
1345  template <typename memfunT, typename arg1T>
1346  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1347  send(const keyT &key, memfunT memfun, const arg1T &arg1) const
1348  {
1349  return const_cast<containerT *>(this)->send(key, memfun, arg1);
1350  }
1351 
1352  /// Sends message "resultT memfun(arg1T,arg2T) const" to item (non-blocking comm if remote)
1353 
1354  /// The method executes with a write lock on the item.
1355  template <typename memfunT, typename arg1T, typename arg2T>
1356  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1357  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2) const
1358  {
1359  return const_cast<containerT *>(this)->send(key, memfun, arg1, arg2);
1360  }
1361 
1362  /// Sends message "resultT memfun(arg1T,arg2T,arg3T) const" to item (non-blocking comm if remote)
1363 
1364  /// The method executes with a write lock on the item.
1365  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T>
1366  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1367  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3) const
1368  {
1369  return const_cast<containerT *>(this)->send(key, memfun, arg1, arg2, arg3);
1370  }
1371 
1372  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T) const" to item (non-blocking comm if remote)
1373 
1374  /// The method executes with a write lock on the item.
1375  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T>
1376  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1377  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4) const
1378  {
1379  return const_cast<containerT *>(this)->send(key, memfun, arg1, arg2, arg3, arg4);
1380  }
1381 
1382  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T) const" to item (non-blocking comm if remote)
1383 
1384  /// The method executes with a write lock on the item.
1385  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T>
1386  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1387  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5) const
1388  {
1389  return const_cast<containerT *>(this)->send(key, memfun, arg1, arg2, arg3, arg4, arg5);
1390  }
1391 
1392  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T) const" to item (non-blocking comm if remote)
1393 
1394  /// The method executes with a write lock on the item.
1395  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T>
1396  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1397  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3,
1398  const arg4T &arg4, const arg5T &arg5, const arg6T &arg6) const
1399  {
1400  return const_cast<containerT *>(this)->send(key, memfun, arg1, arg2, arg3, arg4, arg5, arg6);
1401  }
1402 
1403  /// Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T) const" to item (non-blocking comm if remote)
1404 
1405  /// The method executes with a write lock on the item.
1406  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T, typename arg7T>
1407  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1408  send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3,
1409  const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7) const
1410  {
1411  return const_cast<containerT *>(this)->send(key, memfun, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
1412  }
1413 
1414  /// Adds task "resultT memfun()" in process owning item (non-blocking comm if remote)
1415 
1416  /// If item does not exist it is made with the default constructor.
1417  ///
1418  /// Future arguments for local tasks can generate dependencies, but for remote
1419  /// tasks all futures must be ready.
1420  ///
1421  /// Returns a future result (Future<void> may be ignored).
1422  ///
1423  /// The method executes with a write lock on the item.
1424  template <typename memfunT>
1425  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1426  task(const keyT &key, memfunT memfun, const TaskAttributes &attr = TaskAttributes())
1427  {
1429  MEMFUN_RETURNT(memfunT)
1430  (implT::*itemfun)(const keyT &, memfunT) = &implT::template itemfun<memfunT>;
1431  return p->task(owner(key), itemfun, key, memfun, attr);
1432  }
1433 
1434  /// Adds task "resultT memfun(arg1T)" in process owning item (non-blocking comm if remote)
1435 
1436  /// If item does not exist it is made with the default constructor.
1437  ///
1438  /// Future arguments for local tasks can generate dependencies, but for remote
1439  /// tasks all futures must be ready.
1440  ///
1441  /// Returns a future result (Future<void> may be ignored).
1442  ///
1443  /// The method executes with a write lock on the item.
1444  template <typename memfunT, typename arg1T>
1445  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1446  task(const keyT &key, memfunT memfun, const arg1T &arg1, const TaskAttributes &attr = TaskAttributes())
1447  {
1449  typedef REMFUTURE(arg1T) a1T;
1450  MEMFUN_RETURNT(memfunT)
1451  (implT::*itemfun)(const keyT &, memfunT, const a1T &) = &implT::template itemfun<memfunT, a1T>;
1452  return p->task(owner(key), itemfun, key, memfun, arg1, attr);
1453  }
1454 
1455  /// Adds task "resultT memfun(arg1T,arg2T)" in process owning item (non-blocking comm if remote)
1456 
1457  /// If item does not exist it is made with the default constructor.
1458  ///
1459  /// Future arguments for local tasks can generate dependencies, but for remote
1460  /// tasks all futures must be ready.
1461  ///
1462  /// Returns a future result (Future<void> may be ignored).
1463  ///
1464  /// The method executes with a write lock on the item.
1465  template <typename memfunT, typename arg1T, typename arg2T>
1466  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1467  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const TaskAttributes &attr = TaskAttributes())
1468  {
1470  typedef REMFUTURE(arg1T) a1T;
1471  typedef REMFUTURE(arg2T) a2T;
1472  MEMFUN_RETURNT(memfunT)
1473  (implT::*itemfun)(const keyT &, memfunT, const a1T &, const a2T &) = &implT::template itemfun<memfunT, a1T, a2T>;
1474  return p->task(owner(key), itemfun, key, memfun, arg1, arg2, attr);
1475  }
1476 
1477  /// Adds task "resultT memfun(arg1T,arg2T,arg3T)" in process owning item (non-blocking comm if remote)
1478 
1479  /// If item does not exist it is made with the default constructor.
1480  ///
1481  /// Future arguments for local tasks can generate dependencies, but for remote
1482  /// tasks all futures must be ready.
1483  ///
1484  /// Returns a future result (Future<void> may be ignored).
1485  ///
1486  /// The method executes with a write lock on the item.
1487  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T>
1488  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1489  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const TaskAttributes &attr = TaskAttributes())
1490  {
1492  typedef REMFUTURE(arg1T) a1T;
1493  typedef REMFUTURE(arg2T) a2T;
1494  typedef REMFUTURE(arg3T) a3T;
1495  MEMFUN_RETURNT(memfunT)
1496  (implT::*itemfun)(const keyT &, memfunT, const a1T &, const a2T &, const a3T &) = &implT::template itemfun<memfunT, a1T, a2T, a3T>;
1497  return p->task(owner(key), itemfun, key, memfun, arg1, arg2, arg3, attr);
1498  }
1499 
1500  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T)" in process owning item (non-blocking comm if remote)
1501 
1502  /// If item does not exist it is made with the default constructor.
1503  ///
1504  /// Future arguments for local tasks can generate dependencies, but for remote
1505  /// tasks all futures must be ready.
1506  ///
1507  /// Returns a future result (Future<void> may be ignored).
1508  ///
1509  /// The method executes with a write lock on the item.
1510  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T>
1511  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1512  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const TaskAttributes &attr = TaskAttributes())
1513  {
1515  typedef REMFUTURE(arg1T) a1T;
1516  typedef REMFUTURE(arg2T) a2T;
1517  typedef REMFUTURE(arg3T) a3T;
1518  typedef REMFUTURE(arg4T) a4T;
1519  MEMFUN_RETURNT(memfunT)
1520  (implT::*itemfun)(const keyT &, memfunT, const a1T &, const a2T &, const a3T &, const a4T &) = &implT::template itemfun<memfunT, a1T, a2T, a3T, a4T>;
1521  return p->task(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, attr);
1522  }
1523 
1524  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T)" in process owning item (non-blocking comm if remote)
1525 
1526  /// If item does not exist it is made with the default constructor.
1527  ///
1528  /// Future arguments for local tasks can generate dependencies, but for remote
1529  /// tasks all futures must be ready.
1530  ///
1531  /// Returns a future result (Future<void> may be ignored).
1532  ///
1533  /// The method executes with a write lock on the item.
1534  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T>
1535  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1536  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const TaskAttributes &attr = TaskAttributes())
1537  {
1539  typedef REMFUTURE(arg1T) a1T;
1540  typedef REMFUTURE(arg2T) a2T;
1541  typedef REMFUTURE(arg3T) a3T;
1542  typedef REMFUTURE(arg4T) a4T;
1543  typedef REMFUTURE(arg5T) a5T;
1544  MEMFUN_RETURNT(memfunT)
1545  (implT::*itemfun)(const keyT &, memfunT, const a1T &, const a2T &, const a3T &, const a4T &, const a5T &) = &implT::template itemfun<memfunT, a1T, a2T, a3T, a4T, a5T>;
1546  return p->task(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, arg5, attr);
1547  }
1548 
1549  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T)" in process owning item (non-blocking comm if remote)
1550 
1551  /// If item does not exist it is made with the default constructor.
1552  ///
1553  /// Future arguments for local tasks can generate dependencies, but for remote
1554  /// tasks all futures must be ready.
1555  ///
1556  /// Returns a future result (Future<void> may be ignored).
1557  ///
1558  /// The method executes with a write lock on the item.
1559  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T>
1560  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1561  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const TaskAttributes &attr = TaskAttributes())
1562  {
1564  typedef REMFUTURE(arg1T) a1T;
1565  typedef REMFUTURE(arg2T) a2T;
1566  typedef REMFUTURE(arg3T) a3T;
1567  typedef REMFUTURE(arg4T) a4T;
1568  typedef REMFUTURE(arg5T) a5T;
1569  typedef REMFUTURE(arg6T) a6T;
1570  MEMFUN_RETURNT(memfunT)
1571  (implT::*itemfun)(const keyT &, memfunT, const a1T &, const a2T &, const a3T &, const a4T &, const a5T &, const a6T &) = &implT::template itemfun<memfunT, a1T, a2T, a3T, a4T, a5T, a6T>;
1572  return p->task(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, arg5, arg6, attr);
1573  }
1574 
1575  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T)" in process owning item (non-blocking comm if remote)
1576 
1577  /// If item does not exist it is made with the default constructor.
1578  ///
1579  /// Future arguments for local tasks can generate dependencies, but for remote
1580  /// tasks all futures must be ready.
1581  ///
1582  /// Returns a future result (Future<void> may be ignored).
1583  ///
1584  /// The method executes with a write lock on the item.
1585  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T, typename arg7T>
1586  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1587  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7, const TaskAttributes &attr = TaskAttributes())
1588  {
1590  typedef REMFUTURE(arg1T) a1T;
1591  typedef REMFUTURE(arg2T) a2T;
1592  typedef REMFUTURE(arg3T) a3T;
1593  typedef REMFUTURE(arg4T) a4T;
1594  typedef REMFUTURE(arg5T) a5T;
1595  typedef REMFUTURE(arg6T) a6T;
1596  typedef REMFUTURE(arg7T) a7T;
1597  MEMFUN_RETURNT(memfunT)
1598  (implT::*itemfun)(const keyT &, memfunT, const a1T &, const a2T &, const a3T &, const a4T &, const a5T &, const a6T &, const a7T &) = &implT::template itemfun<memfunT, a1T, a2T, a3T, a4T, a5T, a6T, a7T>;
1599  return p->task(owner(key), itemfun, key, memfun, arg1, arg2, arg3, arg4, arg5, arg6, arg7, attr);
1600  }
1601 
1602  /// Adds task "resultT memfun() const" in process owning item (non-blocking comm if remote)
1603 
1604  /// The method executes with a write lock on the item.
1605  template <typename memfunT>
1606  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1607  task(const keyT &key, memfunT memfun, const TaskAttributes &attr = TaskAttributes()) const
1608  {
1609  return const_cast<containerT *>(this)->task(key, memfun, attr);
1610  }
1611 
1612  /// Adds task "resultT memfun(arg1T) const" in process owning item (non-blocking comm if remote)
1613 
1614  /// The method executes with a write lock on the item.
1615  template <typename memfunT, typename arg1T>
1616  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1617  task(const keyT &key, memfunT memfun, const arg1T &arg1, const TaskAttributes &attr = TaskAttributes()) const
1618  {
1619  return const_cast<containerT *>(this)->task(key, memfun, arg1, attr);
1620  }
1621 
1622  /// Adds task "resultT memfun(arg1T,arg2T) const" in process owning item (non-blocking comm if remote)
1623 
1624  /// The method executes with a write lock on the item.
1625  template <typename memfunT, typename arg1T, typename arg2T>
1626  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1627  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const TaskAttributes &attr = TaskAttributes()) const
1628  {
1629  return const_cast<containerT *>(this)->task(key, memfun, arg1, arg2, attr);
1630  }
1631 
1632  /// Adds task "resultT memfun(arg1T,arg2T,arg3T) const" in process owning item (non-blocking comm if remote)
1633 
1634  /// The method executes with a write lock on the item.
1635  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T>
1636  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1637  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const TaskAttributes &attr = TaskAttributes()) const
1638  {
1639  return const_cast<containerT *>(this)->task(key, memfun, arg1, arg2, arg3, attr);
1640  }
1641 
1642  /// Adds task "resultT memfun(arg1T,arg2T,arg3T, arg4T) const" in process owning item (non-blocking comm if remote)
1643 
1644  /// The method executes with a write lock on the item.
1645  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T>
1646  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1647  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const TaskAttributes &attr = TaskAttributes()) const
1648  {
1649  return const_cast<containerT *>(this)->task(key, memfun, arg1, arg2, arg3, arg4, attr);
1650  }
1651 
1652  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T) const" in process owning item (non-blocking comm if remote)
1653 
1654  /// The method executes with a write lock on the item.
1655  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T>
1656  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1657  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const TaskAttributes &attr = TaskAttributes()) const
1658  {
1659  return const_cast<containerT *>(this)->task(key, memfun, arg1, arg2, arg3, arg4, arg5, attr);
1660  }
1661 
1662  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T) const" in process owning item (non-blocking comm if remote)
1663 
1664  /// The method executes with a write lock on the item.
1665  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T>
1666  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1667  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const TaskAttributes &attr = TaskAttributes()) const
1668  {
1669  return const_cast<containerT *>(this)->task(key, memfun, arg1, arg2, arg3, arg4, arg5, arg6, attr);
1670  }
1671 
1672  /// Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T) const" in process owning item (non-blocking comm if remote)
1673 
1674  /// The method executes with a write lock on the item.
1675  template <typename memfunT, typename arg1T, typename arg2T, typename arg3T, typename arg4T, typename arg5T, typename arg6T, typename arg7T>
1676  Future<REMFUTURE(MEMFUN_RETURNT(memfunT))>
1677  task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7, const TaskAttributes &attr = TaskAttributes()) const
1678  {
1679  return const_cast<containerT *>(this)->task(key, memfun, arg1, arg2, arg3, arg4, arg5, arg6, arg7, attr);
1680  }
1681 
1682  /// (de)Serialize --- *Local* data only to/from anything *except* Buffer*Archive and Parallel*Archive
1683 
1684  /// Advisable for *you* to fence before and after this to ensure consistency
1685  template <typename Archive>
1686  void serialize(const Archive &ar)
1687  {
1688  //
1689  // !! If you change the format of this stream make sure that
1690  // !! the parallel in/out archive below is compatible
1691  //
1692  const long magic = 5881828; // Sitar Indian restaurant in Knoxville
1693  unsigned long count = 0;
1695 
1696  if (Archive::is_output_archive)
1697  {
1698  ar & magic;
1699  for (iterator it = begin(); it != end(); ++it)
1700  count++;
1701  ar & count;
1702  for (iterator it = begin(); it != end(); ++it)
1703  ar &*it;
1704  }
1705  else
1706  {
1707  long cookie = 0l;
1708  ar & cookie;
1709  MADNESS_ASSERT(cookie == magic);
1710  ar & count;
1711  while (count--)
1712  {
1713  pairT datum;
1714  ar & datum;
1715  replace(datum);
1716  }
1717  }
1718  }
1719 
1720  /// (de)Serialize --- !! ONLY for purpose of interprocess communication
1721 
1722  /// This just writes/reads the unique id to/from the Buffer*Archive.
1724  {
1726  ar &static_cast<WorldObject<implT> *>(p.get());
1727  }
1728 
1729  /// (de)Serialize --- !! ONLY for purpose of interprocess communication
1730 
1731  /// This just writes/reads the unique id to/from the Buffer*Archive.
1733  {
1734  WorldObject<implT> *ptr = nullptr;
1735  ar & ptr;
1736  MADNESS_ASSERT(ptr);
1737 
1738 #ifdef MADNESS_DISABLE_SHARED_FROM_THIS
1739  p.reset(static_cast<implT *>(ptr), [](implT *p_) -> void{});
1740 #else
1741  p = static_cast<implT *>(ptr)->shared_from_this();
1742 #endif // MADNESS_DISABLE_SHARED_FROM_THIS
1743  }
1744 
1745  /// Returns the associated unique id ... must be initialized
1746  const uniqueidT &id() const
1747  {
1749  return p->id();
1750  }
1751 
1752  /// Destructor passes ownership of implementation to world for deferred cleanup
1754  {
1755  detail::deferred_cleanup(p->get_world(), p);
1756  }
1757 
1758  friend void swap<>(WorldContainer &, WorldContainer &);
1759  };
1760 
1761  /// Swaps the content of two WorldContainer objects. It should be called on all nodes.
1762 
1763  /// \ingroup worlddc
1764  template <typename keyT, typename valueT, typename hashfunT>
1766  {
1767  std::swap(dc0.p, dc1.p);
1768  }
1769 
1770  namespace archive
1771  {
1772 
1773  /// Write container to parallel archive
1774 
1775  /// specialization for parallel serialization of a WorldContainer:
1776  /// all threads on each process serialize some values into a buffer, which gets concatenated
1777  /// and finally serialized to localarchive (aka VectorOutputArchive).
1778  template <class keyT, class valueT>
1780  {
1782  {
1783  using localarchiveT = VectorOutputArchive;
1784  const long magic = -5881828; // Sitar Indian restaurant in Knoxville (negative to indicate parallel!)
1785  typedef WorldContainer<keyT, valueT> dcT;
1786  using const_iterator = typename dcT::const_iterator;
1787  int count = t.size(); // Must be INT for MPI and NOT const since we'll do a global sum eventually
1788 
1789  // Strategy:
1790  // 1. Serialize local data to a buffer in parallel over threads
1791  // a) Compute the size of the buffer needed by each task
1792  // b) Sum sizes and allocate the buffer of exact sizes needed for all threads
1793  // c) Serialize the data into the buffer in parallel over threads
1794  // 2. Gather all buffers to process 0
1795 
1796  World *world = ar.get_world();
1797  world->gop.fence(); // Global fence here
1798 
1799  class op_inspector : public TaskInterface
1800  {
1801  const_iterator start, end;
1802  size_t &size;
1803 
1804  public:
1805  op_inspector(const_iterator start, const_iterator end, size_t &size)
1806  : start(start), end(end), size(size) {}
1807  void run(World &world)
1808  {
1810  for (const_iterator it = start; it != end; ++it)
1811  bo &*it;
1812  size = bo.size();
1813  }
1814  };
1815 
1816  class op_executor : public TaskInterface
1817  {
1818  const_iterator start, end;
1819  unsigned char *buf;
1820  const size_t size;
1821 
1822  public:
1823  op_executor(const_iterator start, const_iterator end, unsigned char *buf, size_t size)
1824  : start(start), end(end), buf(buf), size(size) {}
1825  void run(World &world)
1826  {
1827  BufferOutputArchive bo(buf, size);
1828  for (const_iterator it = start; it != end; ++it)
1829  {
1830  bo &*it;
1831  }
1832  MADNESS_CHECK(size == bo.size());
1833  }
1834  };
1835 
1836  // No need for LOCAL fence here since only master thread is busy
1837  double wall0 = wall_time();
1838  const size_t ntasks = std::min(size_t(count), std::max(size_t(1), ThreadPool::size()));
1839  size_t local_size = 0;
1840  double wall1 = wall0;
1841  unsigned char* buf = 0;
1842  if (ntasks > 0)
1843  {
1844  const size_t max_items_per_task = (std::max(1, count) - 1) / ntasks + 1;
1845  // Compute the size of the buffer needed by each task
1846  const_iterator starts[ntasks], ends[ntasks];
1847  size_t local_sizes[ntasks];
1848  const_iterator start = t.begin();
1849  size_t nleft = count;
1850  for (size_t taskid = 0; taskid < ntasks; taskid++)
1851  {
1852  const_iterator end = start;
1853  if (taskid == (ntasks - 1))
1854  {
1855  end = t.end();
1856  }
1857  else
1858  {
1859  size_t nitems = std::min(max_items_per_task, nleft);
1860  std::advance(end, max_items_per_task);
1861  nleft -= nitems;
1862  }
1863  starts[taskid] = start;
1864  ends[taskid] = end;
1865  world->taskq.add(new op_inspector(start, end, local_sizes[taskid])); // Be sure to pass iterators by value!!
1866  start = end;
1867  }
1868  world->taskq.fence(); // just need LOCAL fence
1869  wall1 = wall_time();
1870  // if (world->rank() == 0)
1871  // printf("time in op_inspector: %8.4fs\n", wall1 - wall0);
1872  wall0 = wall1;
1873 
1874  // total size over all threads
1875  for (size_t taskid = 0; taskid < ntasks; taskid++)
1876  {
1877  local_size += local_sizes[taskid];
1878  // print("taskid",taskid,"size",local_sizes[taskid]);
1879  }
1880 
1881  // Allocate the buffer for all threads
1882  buf = new unsigned char[local_size];
1883 
1884  // Now execute the serialization
1885  size_t offset = 0;
1886  for (size_t taskid = 0; taskid < ntasks; taskid++)
1887  {
1888  world->taskq.add(new op_executor(starts[taskid], ends[taskid], buf + offset, local_sizes[taskid]));
1889  offset += local_sizes[taskid];
1890  }
1891  world->taskq.fence(); // just need LOCAL fence
1892 
1893  wall1 = wall_time();
1894  // if (world->rank() == 0)
1895  // printf("time in op_executor: %8.4fs\n", wall1 - wall0);
1896  wall0 = wall1;
1897  }
1898  // VERify that the serialization worked!!
1899  // {
1900  // BufferInputArchive bi(buf, local_size);
1901  // for (int item=0; item<count; item++) {
1902  // std::pair<keyT, valueT> datum;
1903  // bi & datum;
1904  // print("deserializing",datum.first);
1905  // }
1906  // }
1907 
1908  // Gather all buffers to process 0
1909  // first gather all of the sizes and counts to a vector in process 0
1910  const int size = local_size;
1911  std::vector<int> sizes(world->size());
1912  MPI_Gather(&size, 1, MPI_INT, sizes.data(), 1, MPI_INT, 0, world->mpi.comm().Get_mpi_comm());
1913  world->gop.sum(count); // just need total number of elements
1914 
1915  // print("time 3",wall_time());
1916  // build the cumulative sum of sizes
1917  std::vector<int> offsets(world->size());
1918  offsets[0] = 0;
1919  for (int i = 1; i < world->size(); ++i)
1920  offsets[i] = offsets[i - 1] + sizes[i - 1];
1921  size_t total_size = offsets.back() + sizes.back();
1922  // if (world->rank() == 0)
1923  // print("total_size", total_size);
1924 
1925  // print("time 4",wall_time());
1926  // gather the vector of data v from each process to process 0
1927  unsigned char *all_data = 0;
1928  if (world->rank() == 0)
1929  {
1930  all_data = new unsigned char[total_size];
1931  }
1932  MPI_Gatherv(buf, local_size, MPI_BYTE, all_data, sizes.data(), offsets.data(), MPI_BYTE, 0, world->mpi.comm().Get_mpi_comm());
1933 
1934  wall1 = wall_time();
1935  // if (world->rank() == 0)
1936  // printf("time in gather+gatherv: %8.4fs\n", wall1 - wall0);
1937  wall0 = wall1;
1938 
1939  delete[] buf;
1940 
1941  // print("time 5",wall_time());
1942  if (world->rank() == 0)
1943  {
1944  auto &localar = ar.local_archive();
1945  localar & magic & 1; // 1 client
1946  // localar & t;
1948  localar & -magic &(unsigned long)(count);
1949  localar.store(all_data, total_size);
1951  wall1 = wall_time();
1952  // if (world->rank() == 0)
1953  // printf("time in final copy on node 0: %8.4fs\n", wall1 - wall0);
1954 
1955  delete[] all_data;
1956  }
1957  world->gop.fence();
1958  // print("time 6",wall_time());
1959  }
1960  };
1961 
1962  /// Write container to parallel archive with optional fence
1963 
1964  /// \ingroup worlddc
1965  /// Each node (process) is served by a designated IO node.
1966  /// The IO node has a binary local file archive to which is
1967  /// first written a cookie and the number of servers. The IO
1968  /// node then loops thru all of its clients and in turn tells
1969  /// each to write its data over an MPI stream, which is copied
1970  /// directly to the output file. The stream contents are then
1971  /// cookie, no. of clients, foreach client (usual sequential archive).
1972  ///
1973  /// If ar.dofence() is true (default) fence is invoked before and
1974  /// after the IO. The fence is optional but it is of course
1975  /// necessary to be sure that all updates have completed
1976  /// before doing IO, and that all IO has completed before
1977  /// subsequent modifications. Also, there is always at least
1978  /// some synchronization between a client and its IO server.
1979  template <class keyT, class valueT, class localarchiveT>
1981  {
1983  {
1984  const long magic = -5881828; // Sitar Indian restaurant in Knoxville (negative to indicate parallel!)
1985  typedef WorldContainer<keyT, valueT> dcT;
1986  // typedef typename dcT::const_iterator iterator; // unused?
1987  typedef typename dcT::pairT pairT;
1988  World *world = ar.get_world();
1989  Tag tag = world->mpi.unique_tag();
1990  ProcessID me = world->rank();
1991  if (ar.dofence())
1992  world->gop.fence();
1993  if (ar.is_io_node())
1994  {
1995  auto &localar = ar.local_archive();
1996  localar & magic & ar.num_io_clients();
1997  for (ProcessID p = 0; p < world->size(); ++p)
1998  {
1999  if (p == me)
2000  {
2001  localar & t;
2002  }
2003  else if (ar.io_node(p) == me)
2004  {
2005  world->mpi.Send(int(1), p, tag); // Tell client to start sending
2007  long cookie = 0l;
2008  unsigned long count = 0ul;
2009 
2011 
2012  source & cookie & count;
2013  localar & cookie & count;
2014  while (count--)
2015  {
2016  pairT datum;
2017  source & datum;
2018  localar & datum;
2019  }
2020 
2022  }
2023  }
2024  }
2025  else
2026  {
2027  ProcessID p = ar.my_io_node();
2028  int flag;
2029  world->mpi.Recv(flag, p, tag);
2030  MPIOutputArchive dest(*world, p);
2031  dest & t;
2032  dest.flush();
2033  }
2034  if (ar.dofence())
2035  world->gop.fence();
2036  }
2037  };
2038 
2039  template <class keyT, class valueT, class localarchiveT>
2041  {
2042  /// Read container from parallel archive
2043 
2044  /// \ingroup worlddc
2045  /// See store method above for format of file content.
2046  /// !!! We presently ASSUME that the number of writers and readers are
2047  /// the same. This is frustrating but not a show stopper since you
2048  /// can always run a separate job to copy to a different number.
2049  ///
2050  /// The IO node simply reads all data and inserts entries.
2052  {
2053  const long magic = -5881828; // Sitar Indian restaurant in Knoxville (negative to indicate parallel!)
2054  // typedef WorldContainer<keyT,valueT> dcT; // unused
2055  // typedef typename dcT::iterator iterator; // unused
2056  // typedef typename dcT::pairT pairT; // unused
2057  World *world = ar.get_world();
2058  if (ar.dofence())
2059  world->gop.fence();
2060  if (ar.is_io_node())
2061  {
2062  long cookie = 0l;
2063  int nclient = 0;
2064  auto &localar = ar.local_archive();
2065  localar & cookie & nclient;
2066  MADNESS_CHECK(cookie == magic);
2067  while (nclient--)
2068  {
2069  localar & t;
2070  }
2071  }
2072  if (ar.dofence())
2073  world->gop.fence();
2074  }
2075  };
2076  }
2077 
2078 }
2079 
2080 ///@}
2081 
2082 #endif // MADNESS_WORLD_WORLDDC_H__INCLUDED
int unique_tag()
Returns a unique tag for temporary use (1023<tag<4095)
Definition: safempi.h:830
MPI_Comm & Get_mpi_comm() const
Definition: safempi.h:709
size_t size() const
Definition: worldhashmap.h:560
hashfunT & get_hash() const
Definition: worldhashmap.h:595
std::pair< iterator, bool > insert(const datumT &datum)
Definition: worldhashmap.h:468
iterator begin()
Definition: worldhashmap.h:571
void erase(const iterator &it)
Definition: worldhashmap.h:507
bool try_erase(const keyT &key)
Definition: worldhashmap.h:502
iterator end()
Definition: worldhashmap.h:583
iterator find(const keyT &key)
Definition: worldhashmap.h:524
void clear()
Definition: worldhashmap.h:556
Implements the functionality of futures.
Definition: future.h:74
A future is a possibly yet unevaluated value.
Definition: future.h:373
remote_refT remote_ref(World &world) const
Returns a structure used to pass references to another process.
Definition: future.h:675
Definition: worldhashmap.h:330
iterator for hash
Definition: worldhashmap.h:188
Range, vaguely a la Intel TBB, to encapsulate a random-access, STL-like start and end iterator with c...
Definition: range.h:64
iteratorT iterator
Alias for the iterator type.
Definition: range.h:71
Simple structure used to manage references/pointers to remote instances.
Definition: worldref.h:395
Contains attributes of a task.
Definition: thread.h:323
All world tasks must be derived from this public interface.
Definition: taskfn.h:69
static std::size_t size()
Returns the number of threads in the pool.
Definition: thread.h:1413
Internal implementation of distributed container to facilitate shallow copy.
Definition: worlddc.h:394
void erase(const keyT &key)
Definition: worlddc.h:603
WorldContainerIterator< internal_iteratorT > iterator
Definition: worlddc.h:409
bool find(const_accessor &acc, const keyT &key) const
Definition: worlddc.h:691
internal_containerT::accessor accessor
Definition: worlddc.h:406
void find_handler(ProcessID requestor, const keyT &key, const RemoteReference< FutureImpl< iterator >> &ref)
Handles find request.
Definition: worlddc.h:433
bool probe(const keyT &key) const
Definition: worlddc.h:554
std::pair< const keyT, valueT > pairT
Definition: worlddc.h:396
hashfunT & get_hash() const
Definition: worlddc.h:542
bool insert_const_acc(const_accessor &acc, const keyT &key)
Definition: worlddc.h:592
WorldContainerIterator< internal_const_iteratorT > const_iteratorT
Definition: worlddc.h:410
bool find(accessor &acc, const keyT &key)
Definition: worlddc.h:684
WorldContainerImpl(World &world, const std::shared_ptr< WorldDCPmapInterface< keyT >> &pm, const hashfunT &hf)
Definition: worlddc.h:469
void redistribute_phase1(const std::shared_ptr< WorldDCPmapInterface< keyT >> &newpmap)
Definition: worlddc.h:788
void find_failure_handler(const RemoteReference< FutureImpl< iterator >> &ref)
Handles unsuccessful find response.
Definition: worlddc.h:459
void redistribute_phase2()
Definition: worlddc.h:819
void insert(const pairT &datum)
Definition: worlddc.h:568
WorldContainerIterator< internal_iteratorT > iteratorT
Definition: worlddc.h:408
void clear()
Definition: worlddc.h:598
bool insert_acc(accessor &acc, const keyT &key)
Definition: worlddc.h:586
void reset_pmap_to_local()
Definition: worlddc.h:492
itemfun(const keyT &key, memfunT memfun)
Definition: worlddc.h:701
const pairT const_pairT
Definition: worlddc.h:397
std::vector< keyT > * move_list
Tempoary used to record data that needs redistributing.
Definition: worlddc.h:430
internal_containerT::iterator internal_iteratorT
Definition: worlddc.h:404
WorldContainerImpl< keyT, valueT, hashfunT > implT
Definition: worlddc.h:398
std::size_t size() const
Definition: worlddc.h:563
std::shared_ptr< WorldDCPmapInterface< keyT > > & get_pmap()
Definition: worlddc.h:487
internal_containerT::const_iterator internal_const_iteratorT
Definition: worlddc.h:405
std::shared_ptr< WorldDCPmapInterface< keyT > > pmap
Function/class to map from keys to owning process.
Definition: worlddc.h:427
virtual ~WorldContainerImpl()
Definition: worlddc.h:477
internal_containerT local
Locally owned data.
Definition: worlddc.h:429
ConcurrentHashMap< keyT, valueT, hashfunT > internal_containerT
Definition: worlddc.h:400
Future< iterator > find(const keyT &key)
Definition: worlddc.h:669
const std::shared_ptr< WorldDCPmapInterface< keyT > > & get_pmap() const
Definition: worlddc.h:482
const_iterator begin() const
Definition: worlddc.h:643
void erase(InIter it)
Definition: worlddc.h:619
void replicate(bool fence)
Definition: worlddc.h:501
const_iterator end() const
Definition: worlddc.h:653
bool is_local(const keyT &key) const
Definition: worlddc.h:544
void redistribute_phase3()
Definition: worlddc.h:833
ProcessID owner(const keyT &key) const
Definition: worlddc.h:549
void erase(InIter first, InIter last)
Definition: worlddc.h:627
iterator begin()
Definition: worlddc.h:638
const ProcessID me
My MPI rank.
Definition: worlddc.h:428
iterator end()
Definition: worlddc.h:648
void find_success_handler(const RemoteReference< FutureImpl< iterator >> &ref, const pairT &datum)
Handles successful find response.
Definition: worlddc.h:449
WorldContainerIterator< internal_const_iteratorT > const_iterator
Definition: worlddc.h:411
Future< const_iterator > find(const keyT &key) const
Definition: worlddc.h:658
internal_containerT::const_accessor const_accessor
Definition: worlddc.h:407
Iterator for distributed container wraps the local iterator.
Definition: worlddc.h:244
WorldContainerIterator(const WorldContainerIterator &other)
Definition: worlddc.h:273
WorldContainerIterator(const internal_iteratorT &it)
Initializes from a local iterator.
Definition: worlddc.h:263
WorldContainerIterator & operator=(const WorldContainerIterator &other)
Assignment.
Definition: worlddc.h:292
std::iterator_traits< internal_iteratorT >::iterator_category iterator_category
Definition: worlddc.h:246
void copy(const WorldContainerIterator< iteratorT > &other)
Definition: worlddc.h:364
std::iterator_traits< internal_iteratorT >::pointer pointer
Definition: worlddc.h:249
bool operator==(const WorldContainerIterator &other) const
Determines if two iterators are identical.
Definition: worlddc.h:299
value_type * value
holds the remote values
Definition: worlddc.h:255
bool is_cached() const
Returns true if this is non-local or cached value.
Definition: worlddc.h:348
WorldContainerIterator operator++(int)
Definition: worlddc.h:321
WorldContainerIterator & operator++()
Pre-increment of an iterator (i.e., ++it) — local iterators only.
Definition: worlddc.h:314
WorldContainerIterator(const value_type &v)
Initializes to cache a remote value.
Definition: worlddc.h:267
pointer operator->() const
Iterators dereference to std::pair<const keyT,valueT>
Definition: worlddc.h:330
std::iterator_traits< internal_iteratorT >::reference reference
Definition: worlddc.h:250
void serialize(const Archive &)
Definition: worlddc.h:354
std::iterator_traits< internal_iteratorT >::value_type value_type
Definition: worlddc.h:247
WorldContainerIterator(const WorldContainerIterator< iteratorT > &other)
Definition: worlddc.h:280
WorldContainerIterator()
Default constructor makes a local uninitialized value.
Definition: worlddc.h:259
internal_iteratorT it
Iterator from local container.
Definition: worlddc.h:253
const internal_iteratorT & get_internal_iterator() const
Private: (or should be) Returns iterator of internal container.
Definition: worlddc.h:342
reference operator*() const
Iterators dereference to std::pair<const keyT,valueT>
Definition: worlddc.h:336
std::iterator_traits< internal_iteratorT >::difference_type difference_type
Definition: worlddc.h:248
bool operator!=(const WorldContainerIterator &other) const
Determines if two iterators are different.
Definition: worlddc.h:306
~WorldContainerIterator()
Definition: worlddc.h:286
Makes a distributed container with specified attributes.
Definition: worlddc.h:866
void process_pending()
Process pending messages.
Definition: worlddc.h:1166
WorldContainer(World &world, bool do_pending=true, const hashfunT &hf=hashfunT())
Makes an initialized, empty container with default data distribution (no communication)
Definition: worlddc.h:904
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4)
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T)" to item (non-blocking comm if remote)
Definition: worlddc.h:1266
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5)
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T)" to item (non-blocking comm if remote)
Definition: worlddc.h:1285
bool find(accessor &acc, const keyT &key)
Write access to LOCAL value by key. Returns true if found, false otherwise (always false for remote).
Definition: worlddc.h:987
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7) const
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T) const" to item (non-blocking...
Definition: worlddc.h:1408
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T,arg2T)" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1467
bool probe(const keyT &key) const
Returns true if local data is immediately available (no communication)
Definition: worlddc.h:1024
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5) const
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T) const" to item (non-blocking comm if rem...
Definition: worlddc.h:1387
const_iterator begin() const
Returns an iterator to the beginning of the local data (no communication)
Definition: worlddc.h:1077
void replace(const keyT &key, const valueT &value)
Inserts/replaces key+value pair (non-blocking communication if key not local)
Definition: worlddc.h:981
bool insert(const_accessor &acc, const keyT &key)
Read access to LOCAL value by key. Returns true if inserted, false if already exists (throws if remot...
Definition: worlddc.h:1008
iterator begin()
Returns an iterator to the beginning of the local data (no communication)
Definition: worlddc.h:1070
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T,arg2T) const" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1627
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T) const" in process owning item (non-blo...
Definition: worlddc.h:1667
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun) const
Sends message "resultT memfun() const" to item (non-blocking comm if remote)
Definition: worlddc.h:1337
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3)
Sends message "resultT memfun(arg1T,arg2T,arg3T)" to item (non-blocking comm if remote)
Definition: worlddc.h:1247
Future< const_iterator > const_futureT
Definition: worlddc.h:876
ProcessID owner(const keyT &key) const
Returns processor that logically owns key (no communication)
Definition: worlddc.h:1034
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3) const
Sends message "resultT memfun(arg1T,arg2T,arg3T) const" to item (non-blocking comm if remote)
Definition: worlddc.h:1367
implT::const_iterator const_iterator
Definition: worlddc.h:872
WorldContainer()
Makes an uninitialized container (no communication)
Definition: worlddc.h:892
void serialize(const archive::BufferOutputArchive &ar)
(de)Serialize — !! ONLY for purpose of interprocess communication
Definition: worlddc.h:1723
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, const memfunT &memfun, const arg1T &arg1)
Sends message "resultT memfun(arg1T)" to item (non-blocking comm if remote)
Definition: worlddc.h:1202
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4) const
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T) const" to item (non-blocking comm if remote)
Definition: worlddc.h:1377
void replicate(bool fence=true)
replicates this WorldContainer on all ProcessIDs
Definition: worlddc.h:968
virtual ~WorldContainer()
Destructor passes ownership of implementation to world for deferred cleanup.
Definition: worlddc.h:1753
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2) const
Sends message "resultT memfun(arg1T,arg2T) const" to item (non-blocking comm if remote)
Definition: worlddc.h:1357
const std::shared_ptr< WorldDCPmapInterface< keyT > > & get_pmap() const
Returns shared pointer to the process mapping.
Definition: worlddc.h:1142
void erase(const keyT &key)
Erases entry from container (non-blocking comm if remote)
Definition: worlddc.h:1105
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T,arg2T,arg3T)" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1489
void reset_pmap_to_local()
Returns shared pointer to the process mapping.
Definition: worlddc.h:1149
std::shared_ptr< WorldDCPmapInterface< keyT > > & get_impl()
Definition: worlddc.h:961
Future< iterator > find(const keyT &key)
Returns a future iterator (non-blocking communication if key not local)
Definition: worlddc.h:1052
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2)
Sends message "resultT memfun(arg1T,arg2T)" to item (non-blocking comm if remote)
Definition: worlddc.h:1225
WorldContainerImpl< keyT, valueT, hashfunT > implT
Definition: worlddc.h:869
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T)" in process owning item (non-blocking ...
Definition: worlddc.h:1561
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T)" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1446
void replace(const pairT &datum)
Inserts/replaces key+value pair (non-blocking communication if key not local)
Definition: worlddc.h:974
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun() const" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1607
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T) const" in process owning item (non-blocking ...
Definition: worlddc.h:1657
iterator end()
Returns an iterator past the end of the local data (no communication)
Definition: worlddc.h:1084
WorldContainer(World &world, const std::shared_ptr< WorldDCPmapInterface< keyT >> &pmap, bool do_pending=true, const hashfunT &hf=hashfunT())
Makes an initialized, empty container (no communication)
Definition: worlddc.h:920
void replace(input_iterator &start, input_iterator &end)
Inserts pairs (non-blocking communication if key(s) not local)
Definition: worlddc.h:1016
bool insert(accessor &acc, const keyT &key)
Write access to LOCAL value by key. Returns true if inserted, false if already exists (throws if remo...
Definition: worlddc.h:1001
Future< iterator > futureT
Definition: worlddc.h:875
Future< const_iterator > find(const keyT &key) const
Returns a future iterator (non-blocking communication if key not local)
Definition: worlddc.h:1063
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun()" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1426
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7)
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T)" to item (non-blocking comm ...
Definition: worlddc.h:1323
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T) const" in process owning item (non-blocking comm if remote)
Definition: worlddc.h:1617
void erase(const iterator &it)
Erases entry corresponding to local iterator (no communication)
Definition: worlddc.h:1112
void erase(const iterator &start, const iterator &finish)
Erases range defined by local iterators (no communication)
Definition: worlddc.h:1119
WorldContainer(const WorldContainer &other)
Copy constructor is shallow (no communication)
Definition: worlddc.h:934
World & get_world() const
Returns the world associated with this container.
Definition: worlddc.h:955
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T,arg2T,arg3T) const" in process owning item (non-blocking comm if remo...
Definition: worlddc.h:1637
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T)" in process owning item (non-blocking comm i...
Definition: worlddc.h:1536
implT::iterator iterator
Definition: worlddc.h:871
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T,arg2T,arg3T, arg4T) const" in process owning item (non-blocking comm ...
Definition: worlddc.h:1647
implT::pairT pairT
Definition: worlddc.h:870
std::size_t size() const
Returns the number of local entries (no communication)
Definition: worlddc.h:1135
hashfunT & get_hash() const
Returns a reference to the hashing functor.
Definition: worlddc.h:1155
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1) const
Sends message "resultT memfun(arg1T) const" to item (non-blocking comm if remote)
Definition: worlddc.h:1347
bool find(const_accessor &acc, const keyT &key) const
Read access to LOCAL value by key. Returns true if found, false otherwise (always false for remote).
Definition: worlddc.h:994
void serialize(const archive::BufferInputArchive &ar)
(de)Serialize — !! ONLY for purpose of interprocess communication
Definition: worlddc.h:1732
bool is_local(const keyT &key) const
Returns true if the key maps to the local processor (no communication)
Definition: worlddc.h:1041
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6) const
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T) const" to item (non-blocking comm ...
Definition: worlddc.h:1397
const uniqueidT & id() const
Returns the associated unique id ... must be initialized.
Definition: worlddc.h:1746
const_iterator end() const
Returns an iterator past the end of the local data (no communication)
Definition: worlddc.h:1091
void serialize(const Archive &ar)
(de)Serialize — Local data only to/from anything except Buffer*Archive and Parallel*Archive
Definition: worlddc.h:1686
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7, const TaskAttributes &attr=TaskAttributes()) const
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T) const" in process owning item (n...
Definition: worlddc.h:1677
containerT & operator=(const containerT &other)
Assignment is shallow (no communication)
Definition: worlddc.h:944
void clear()
Clears all local data (no communication)
Definition: worlddc.h:1128
implT::const_accessor const_accessor
Definition: worlddc.h:874
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T)" in process owning item (non-blocking comm if remo...
Definition: worlddc.h:1512
std::shared_ptr< implT > p
Definition: worlddc.h:879
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> task(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6, const arg7T &arg7, const TaskAttributes &attr=TaskAttributes())
Adds task "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T,arg7T)" in process owning item (non-blo...
Definition: worlddc.h:1587
Future< REMFUTURE(MEMFUN_RETURNT(memfunT))> send(const keyT &key, memfunT memfun, const arg1T &arg1, const arg2T &arg2, const arg3T &arg3, const arg4T &arg4, const arg5T &arg5, const arg6T &arg6)
Sends message "resultT memfun(arg1T,arg2T,arg3T,arg4T,arg5T,arg6T)" to item (non-blocking comm if rem...
Definition: worlddc.h:1304
void check_initialized() const
Definition: worlddc.h:881
Future< MEMFUN_RETURNT(memfunT)> send(const keyT &key, memfunT memfun)
Sends message "resultT memfun()" to item (non-blocking comm if remote)
Definition: worlddc.h:1183
WorldContainer< keyT, valueT, hashfunT > containerT
Definition: worlddc.h:868
implT::accessor accessor
Definition: worlddc.h:873
Default process map is "random" using madness::hash(key)
Definition: worlddc.h:203
ProcessID owner(const keyT &key) const
Maps key to processor.
Definition: worlddc.h:214
WorldDCDefaultPmap(World &world, const hashfunT &hf=hashfunT())
Definition: worlddc.h:209
const int nproc
Definition: worlddc.h:205
hashfunT hashfun
Definition: worlddc.h:206
Local process map will always return the current process as owner.
Definition: worlddc.h:227
WorldDCLocalPmap(World &world)
Definition: worlddc.h:232
ProcessID me
Definition: worlddc.h:229
ProcessID owner(const keyT &key) const
Maps key to processor.
Definition: worlddc.h:233
Interface to be provided by any process map.
Definition: worlddc.h:82
void print_data_sizes(World &world, const std::string msg="") const
Prints size info to std::cout.
Definition: worlddc.h:178
std::size_t global_size(World &world) const
Counts global number of entries in all containers associated with this process map.
Definition: worlddc.h:155
virtual ProcessID owner(const keyT &key) const =0
Maps key to processor.
void deregister_callback(ptrT ptr)
Deregisters object for receipt of redistribute callbacks.
Definition: worlddc.h:111
void register_callback(ptrT ptr)
Registers object for receipt of redistribute callbacks.
Definition: worlddc.h:103
virtual void print() const
Definition: worlddc.h:98
std::size_t local_size() const
Counts local number of entries in all containers associated with this process map.
Definition: worlddc.h:165
std::set< ptrT > ptrs
Definition: worlddc.h:87
void redistribute(World &world, const std::shared_ptr< WorldDCPmapInterface< keyT >> &newpmap)
Invoking this switches all registered objects from this process map to the new one.
Definition: worlddc.h:122
virtual ~WorldDCPmapInterface()
Definition: worlddc.h:96
WorldDCRedistributeInterface< keyT > * ptrT
Definition: worlddc.h:84
virtual std::size_t size() const =0
virtual void redistribute_phase1(const std::shared_ptr< WorldDCPmapInterface< keyT >> &newmap)=0
virtual ~WorldDCRedistributeInterface()
Definition: worlddc.h:74
void broadcast_serializable(objT &obj, ProcessID root)
Broadcast a serializable object.
Definition: worldgop.h:754
void fence(bool debug=false)
Synchronizes all processes in communicator AND globally ensures no pending AM or tasks.
Definition: worldgop.cc:161
void sum(T *buf, size_t nelem)
Inplace global sum while still processing AM & tasks.
Definition: worldgop.h:870
void Send(const T *buf, long lenbuf, int dest, int tag=SafeMPI::DEFAULT_SEND_RECV_TAG) const
Send array of lenbuf elements to process dest.
Definition: worldmpi.h:347
void Recv(T *buf, long lenbuf, int src, int tag) const
Receive data of up to lenbuf elements from process src.
Definition: worldmpi.h:374
SafeMPI::Intracomm & comm()
Returns the associated SafeMPI communicator.
Definition: worldmpi.h:286
Implements most parts of a globally addressable object (via unique ID).
Definition: world_object.h:364
detail::task_result_type< memfnT >::futureT send(ProcessID dest, memfnT memfn) const
Definition: world_object.h:731
World & world
The World this object belongs to. (Think globally, act locally).
Definition: world_object.h:381
detail::task_result_type< memfnT >::futureT task(ProcessID dest, memfnT memfn, const TaskAttributes &attr=TaskAttributes()) const
Sends task to derived class method returnT (this->*memfn)().
Definition: world_object.h:1005
World & get_world() const
Returns a reference to the world.
Definition: world_object.h:717
Future< bool > for_each(const rangeT &range, const opT &op)
Apply op(item) on all items in range.
Definition: world_task_queue.h:572
void add(TaskInterface *t)
Add a new local task, taking ownership of the pointer.
Definition: world_task_queue.h:466
void fence()
Returns after all local tasks have completed.
Definition: world_task_queue.h:1384
A parallel world class.
Definition: world.h:132
WorldTaskQueue & taskq
Task queue.
Definition: world.h:204
ProcessID rank() const
Returns the process rank in this World (same as MPI_Comm_rank()).
Definition: world.h:318
WorldMpiInterface & mpi
MPI interface.
Definition: world.h:202
ProcessID size() const
Returns the number of processes in this World (same as MPI_Comm_size()).
Definition: world.h:328
WorldGopInterface & gop
Global operations.
Definition: world.h:205
bool dofence() const
Check if we should fence around a read/write operation.
Definition: parallel_archive.h:295
Archive & local_archive() const
Returns a reference to the local archive.
Definition: parallel_archive.h:248
World * get_world() const
Returns a pointer to the world.
Definition: parallel_archive.h:130
bool is_io_node() const
Returns true if this node is doing physical I/O.
Definition: parallel_archive.h:122
int num_io_clients() const
Returns the number of I/O clients for this node, including self (zero if not an I/O node).
Definition: parallel_archive.h:114
ProcessID io_node(ProcessID rank) const
Returns the process doing I/O for given node.
Definition: parallel_archive.h:99
ProcessID my_io_node() const
Returns the process doing I/O for this node.
Definition: parallel_archive.h:106
Wraps an archive around a memory buffer for input.
Definition: buffer_archive.h:134
Wraps an archive around a memory buffer for output.
Definition: buffer_archive.h:59
std::size_t size() const
Return the amount of data stored (counted) in the buffer.
Definition: buffer_archive.h:123
Archive allowing buffering, deserialization of data, and point-to-point communication between process...
Definition: mpi_archive.h:180
Archive allowing buffering, serialization of data, and point-to-point communication between processes...
Definition: mpi_archive.h:118
void flush() const
Send all data in the buffer to the destination process.
Definition: mpi_archive.h:158
An archive for storing local or parallel data, wrapping a BinaryFstreamInputArchive.
Definition: parallel_archive.h:366
An archive for storing local or parallel data wrapping a BinaryFstreamOutputArchive.
Definition: parallel_archive.h:321
Objects that implement their own parallel archive interface should derive from this class.
Definition: parallel_archive.h:58
Wraps an archive around an STL vector for output.
Definition: vector_archive.h:55
Class for unique global IDs.
Definition: uniqueid.h:53
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition: derivatives.cc:72
void run(World &world, ansatzT ansatz, const int nuclear_charge, const commandlineparser &parser, const int nstates)
Definition: dirac-hatom.cc:1388
void swap(WorldContainer< keyT, valueT, hashfunT > &, WorldContainer< keyT, valueT, hashfunT > &)
Swaps the content of two WorldContainer objects. It should be called on all nodes.
Definition: worlddc.h:1765
static void load(const ParallelInputArchive< localarchiveT > &ar, WorldContainer< keyT, valueT > &t)
Read container from parallel archive.
Definition: worlddc.h:2051
static const double v
Definition: hatom_sf_dirac.cc:20
#define max(a, b)
Definition: lda.h:51
#define MADNESS_CHECK(condition)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition: madness_exception.h:190
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition: madness_exception.h:119
#define MADNESS_ASSERT(condition)
Assert a condition that should be free of side-effects since in release builds this might be a no-op.
Definition: madness_exception.h:134
Implements archives to serialize data for MPI.
void deferred_cleanup(World &world, const std::shared_ptr< objT > &p, bool assume_p_is_unique=false)
Defer the cleanup of a shared pointer to the end of the next fence.
Definition: deferred_cleanup.h:135
File holds all helper structures necessary for the CC_Operator and CC2 class.
Definition: DFParameters.h:10
void print(const T &t, const Ts &... ts)
Print items to std::cout (items separated by spaces) and terminate with a new line.
Definition: print.h:225
Function< T, NDIM > sum(World &world, const std::vector< Function< T, NDIM > > &f, bool fence=true)
Returns new function — q = sum_i f[i].
Definition: vmra.h:1421
NDIM & f
Definition: mra.h:2416
double wall_time()
Returns the wall time in seconds relative to an arbitrary origin.
Definition: timers.cc:48
void swap(Vector< T, N > &l, Vector< T, N > &r)
Swap the contents of two Vectors.
Definition: vector.h:497
Definition: mraimpl.h:50
void advance(madness::Hash_private::HashIterator< hashT > &it, const distT &dist)
Definition: worldhashmap.h:610
Implements ParallelInputArchive and ParallelOutputArchive for parallel serialization of data.
Definition: worlddc.h:800
bool operator()(typename rangeT::iterator &iterator) const
Definition: worlddc.h:805
implT * impl
Definition: worlddc.h:801
P2Op(const P2Op &p)
Definition: worlddc.h:804
P2Op(implT *impl)
Definition: worlddc.h:803
Range< typename std::vector< keyT >::const_iterator > rangeT
Definition: worlddc.h:802
Default load of an object via serialize(ar, t).
Definition: archive.h:666
static void postamble_store(const Archive &)
By default there is no postamble.
Definition: archive.h:545
static void preamble_store(const Archive &ar)
Serialize a cookie for type checking.
Definition: archive.h:535
static void store(const ParallelOutputArchive< VectorOutputArchive > &ar, const WorldContainer< keyT, valueT > &t)
Definition: worlddc.h:1781
static void store(const ParallelOutputArchive< localarchiveT > &ar, const WorldContainer< keyT, valueT > &t)
Definition: worlddc.h:1982
Default store of an object via serialize(ar, t).
Definition: archive.h:611
#define MPI_INT
Definition: stubmpi.h:78
#define MPI_BYTE
Definition: stubmpi.h:74
std::pair< int, double > valueT
Definition: test_binsorter.cc:6
int me
Definition: test_binsorter.cc:10
const double offset
Definition: testfuns.cc:143
double source(const coordT &r)
Definition: testperiodic.cc:48
#define REMFUTURE(T)
Macro to determine type of future (by removing wrapping Future template).
Definition: type_traits.h:162
#define MEMFUN_RETURNT(MEMFUN)
Macro to make member function type traits easier to use.
Definition: type_traits.h:697
Defines and implements WorldObject.
Defines and implements a concurrent hashmap.
int ProcessID
Used to clearly identify process number/rank.
Definition: worldtypes.h:43
int Tag
Used to clearly identify message tag/type.
Definition: worldtypes.h:44