1#ifndef SRC_APPS_CHEM_EXCHANGEOPERATOR_H_
2#define SRC_APPS_CHEM_EXCHANGEOPERATOR_H_
32 const long granularity_level) {
34 const long nsw = std::max<long>(1, nsubworld);
35 const long level = std::max<long>(1, granularity_level);
36 return std::min<long>(level * nsw,
long(n));
46 const long granularity_level) {
47 std::vector<Batch_1D> out;
48 if (n == 0)
return out;
50 const long bs_floor = long(n) / nbatch;
51 const long rem = long(n) - bs_floor * nbatch;
53 for (
long b = 0;
b < nbatch; ++
b) {
54 const long sz = bs_floor + (
b < rem ? 1 : 0);
55 out.emplace_back(begin, begin + sz);
78inline std::vector<std::pair<Batch_1D, Batch_1D>>
82 std::vector<std::pair<Batch_1D, Batch_1D>> out;
83 out.reserve(columns.size() * rows.size());
84 for (
const auto&
c : columns)
85 for (
const auto& r : rows) out.emplace_back(
c, r);
95template<
typename T, std::
size_t NDIM>
98 if (
a.size() !=
b.size())
return false;
99 for (std::size_t i = 0; i <
a.size(); ++i)
111inline std::map<std::pair<long,long>,
long>
113 std::map<std::pair<long,long>,
long> owner;
114 if (ncolumn <= 0 or nrow <= 0 or nworker <= 0)
return owner;
115 for (
long c = 0;
c < ncolumn; ++
c)
116 for (
long r = 0; r < nrow; ++r) owner[{
c, r}] =
c % nworker;
125 const long a = std::max(i, j),
b = std::min(i, j);
126 return a * (
a + 1) / 2 +
b;
147inline std::map<std::pair<long,long>,
long>
149 std::map<std::pair<long,long>,
long> owner;
150 if (n <= 0 or M <= 0)
return owner;
151 auto eligible = [n](
long i,
long j,
long t) ->
bool {
152 if (i == j)
return (i % n) == t;
153 return ((i % n) == t) or ((j % n) == t);
156 const std::size_t ntask = std::size_t(M) * std::size_t(M + 1) / 2;
157 std::vector<std::pair<long,long>> tasks;
158 tasks.reserve(ntask);
159 for (
long i = 0; i < M; ++i)
160 for (
long j = 0; j <= i; ++j)
161 tasks.emplace_back(i, j);
163 std::vector<std::vector<std::pair<long,long>>> T(n);
170 std::vector<char> taken(ntask, 0);
171 std::vector<std::size_t> cursor(n, 0);
172 std::size_t placed = 0;
173 long t = 0, misses = 0;
174 while (placed < ntask and misses < n) {
175 std::size_t&
c = cursor[t];
176 while (
c < ntask and (taken[
c] or not eligible(tasks[
c].first, tasks[
c].second, t))) ++
c;
179 T[t].push_back(tasks[
c]);
189 for (std::size_t idx = 0; idx < ntask; ++idx)
190 if (not taken[idx]) T[tasks[idx].first % n].push_back(tasks[idx]);
193 for (
long iter = 0; iter < 10000; ++iter) {
194 long big = 0,
small = 0;
195 for (
long p = 1;
p < n; ++
p) {
196 if (
long(T[
p].size()) >
long(T[big].size())) big =
p;
197 if (
long(T[
p].size()) <
long(T[
small].size()))
small =
p;
199 if (
long(T[big].size()) -
long(T[
small].size()) <= 1)
break;
201 std::vector<long> order(n);
202 for (
long p = 0;
p < n; ++
p) order[
p] =
p;
203 std::sort(order.begin(), order.end(),
204 [&T](
long a,
long b){ return T[a].size() > T[b].size(); });
205 for (
long bi = 0; bi < n and not moved; ++bi) {
206 for (
long si = n - 1; si > bi and not moved; --si) {
207 const long bt = order[bi], st = order[si];
208 if (
long(T[bt].size()) -
long(T[st].size()) <= 1)
continue;
209 for (
long idx = 0; idx < long(T[bt].size()); ++idx) {
210 const auto& tk = T[bt][idx];
211 if (tk.first == tk.second)
continue;
212 if (not eligible(tk.first, tk.second, st))
continue;
214 T[bt].erase(T[bt].begin() + idx);
220 if (not moved)
break;
223 for (
long p = 0;
p < n; ++
p)
224 for (
const auto& tk : T[
p])
241inline std::map<std::pair<long,long>,
long>
243 std::map<std::pair<long,long>,
long> owner;
244 if (n <= 0 or M <= 0)
return owner;
245 auto C = [&](
long i,
long j) ->
double {
246 const long t =
exchange_sym_tri(i, j);
return (t <
long(cost.size())) ? cost[t] : 0.0;
248 std::vector<double>
load(n, 0.0);
250 for (
long i = 0; i < M; ++i) {
const long r = i % n;
load[r] +=
C(i,i); owner[{i,i}] = r; }
252 std::vector<std::pair<long,long>> off;
253 off.reserve(std::size_t(M) * std::size_t(M > 0 ? M - 1 : 0) / 2);
254 for (
long i = 0; i < M; ++i)
for (
long j = 0; j < i; ++j) off.emplace_back(i, j);
255 std::sort(off.begin(), off.end(), [&](
const std::pair<long,long>&
A,
const std::pair<long,long>&
B){
256 const double ca = C(A.first,A.second), cb = C(B.first,B.second);
257 return (ca != cb) ? (ca > cb) : (A < B);
259 for (
const auto& [i,j] : off) {
260 const long ri = i % n, rj = j % n;
261 const long r = (ri == rj) ? ri : (
load[ri] <=
load[rj] ? ri : rj);
262 owner[{i,j}] = r;
load[r] +=
C(i,j);
265 for (
long pass = 0; pass < 50; ++pass) {
266 long hot = 0;
for (
long p = 1;
p < n; ++
p)
if (
load[
p] >
load[hot]) hot =
p;
267 bool improved =
false;
268 for (
const auto& [i,j] : off) {
269 auto it = owner.find({i,j});
270 if (it->second != hot)
continue;
271 const long ri = i % n, rj = j % n;
272 if (ri == rj)
continue;
273 const long other = (hot == ri) ? rj : ri;
274 const double c =
C(i,j);
276 load[hot] -=
c;
load[other] +=
c; it->second = other; improved =
true;
279 if (not improved)
break;
318 if (tier == 2)
waited =
true;
324 static const bool on = (std::getenv(
"MAD_EXCH_TASK_PROFILE") !=
nullptr);
335 static const bool on = (std::getenv(
"MAD_EXCH_FORCE_GENERAL") !=
nullptr);
345 static std::ofstream os;
346 if (not os.is_open()) {
347 os.open(
"exch_taskprof.r" + std::to_string(
p.universe_rank) +
".jsonl", std::ios::app);
348 if (not os.is_open())
return;
350 os <<
"{\"task\":" <<
p.task_id
351 <<
",\"rank\":" <<
p.universe_rank
352 <<
",\"subworld\":" <<
p.subworld_id
353 <<
",\"subworld_nrank\":" <<
p.subworld_nrank
354 <<
",\"thresh\":" <<
p.thresh
356 <<
",\"diagonal\":" << (
p.diagonal ?
"true" :
"false")
357 <<
",\"row\":[" <<
p.row_begin <<
"," <<
p.row_end <<
"]"
358 <<
",\"col\":[" <<
p.col_begin <<
"," <<
p.col_end <<
"]"
359 <<
",\"wall_start\":" <<
p.wall_start
360 <<
",\"wall\":" << (
p.wall_end -
p.wall_start)
361 <<
",\"wait_for_data\":" <<
p.wait_for_data_wall
362 <<
",\"compute_wall\":" <<
p.compute_wall
363 <<
",\"compute_cpu\":" <<
p.compute_cpu
364 <<
",\"compute_components_wall\":{"
365 <<
"\"mul1\":" <<
p.mul1_wall <<
",\"apply\":" <<
p.apply_wall
366 <<
",\"mul2\":" <<
p.mul2_wall <<
",\"truncate\":" <<
p.truncate_wall
367 <<
",\"other\":" << (
p.compute_wall -
p.mul1_wall -
p.apply_wall
368 -
p.mul2_wall -
p.truncate_wall)
370 <<
",\"operand_source\":" <<
p.operand_source
371 <<
",\"waited\":" << (
p.waited ?
"true" :
"false")
372 <<
",\"peak_rss_gb\":" <<
p.peak_rss_gb
385 const std::vector<double>& cost) {
386 if (M <= 0 or cost.empty())
return;
388 std::snprintf(
name,
sizeof(
name),
"Ccall%03ld_k%ld.csv", call_index,
k);
389 std::ofstream os(
name);
390 if (not os.is_open())
return;
391 os <<
"# call=" << call_index <<
" k=" <<
k <<
" M=" << M <<
"\n";
393 for (
long i = 0; i < M; ++i)
394 for (
long j = 0; j <= i; ++j) {
395 const long t = i * (i + 1) / 2 + j;
396 if (t <
long(cost.size())) os << i <<
"," << j <<
"," << cost[t] <<
"\n";
414template<
typename T, std::
size_t NDIM>
416 std::size_t
k = 0x5a17ull;
425 std::size_t
k = std::size_t(salt);
442template<
typename keyT,
typename dataT>
450 for (
const auto& s :
slots_)
if (s.key == key)
return true;
456 for (
auto it =
slots_.begin(); it !=
slots_.end(); ++it) {
457 if (it->key == key) {
459 return &
slots_.front().data;
467 slots_.push_front(Slot{key, std::move(
data), pinned});
472 if (not it->pinned) {
slots_.erase(it);
break; }
475 return slots_.front().data;
485 for (
const auto& s :
slots_)
if (not s.pinned) ++
c;
500template <
typename T, std::
size_t NDIM>
505 template <
typename Archive>
515template <
typename T, std::
size_t NDIM>
528 dests_ = std::move(dests);
533 for (
const auto& r : recs) {
537 "exchange finalize: a chunk names a destination this rank has not "
538 "been given, so its reducer targets were never set");
539 typename implT::dcT::accessor acc;
540 dests_[r.f]->get_coeffs().insert(acc, r.key);
541 acc->second.template gaxpy_inplace<T, T>(T(1.0), r.node,
beta_);
561template <
typename T, std::
size_t NDIM>
567 const std::size_t chunk_entries) {
571 std::vector<implT*> dests(dest_vec.size(),
nullptr);
572 for (std::size_t
f = 0;
f < dest_vec.size(); ++
f)
577 std::map<ProcessID, std::vector<recT>> buckets;
578 const std::size_t nf = std::min(src_vec.size(), dest_vec.size());
579 for (std::size_t
f = 0;
f < nf; ++
f) {
580 auto simpl = src_vec[
f].get_impl();
581 auto dimpl = dest_vec[
f].get_impl();
582 if (not simpl or not dimpl)
continue;
583 const implT& dref = *dimpl;
584 for (
auto it = simpl->get_coeffs().begin(); it != simpl->get_coeffs().end(); ++it) {
585 const ProcessID owner = dref.get_coeffs().owner(it->first);
586 std::vector<recT>&
b = buckets[owner];
587 b.push_back(recT{
f, it->first, it->second});
588 if (
b.size() >= chunk_entries) {
594 for (
auto& kv : buckets)
595 if (not kv.second.empty())
600template<
typename T, std::
size_t NDIM>
632 world.gop.sum(resident);
633 world.gop.sum(fetched);
634 world.gop.sum(ahead);
640 j[
"batch_cache_hits"] = long(resident);
641 j[
"batch_cache_misses"] = long(fetched);
642 j[
"batch_prefetch_hits"] = long(ahead);
648 if (
world.rank() == 0) {
649 printf(
" cpu time spent in multiply1 %8.2fs\n", timings[
"multiply1"].
template get<double>());
650 printf(
" cpu time spent in apply %8.2fs\n", timings[
"apply"].
template get<double>());
651 printf(
" cpu time spent in multiply2 %8.2fs\n", timings[
"multiply2"].
template get<double>());
652 printf(
" total wall time %8.2fs\n", timings[
"total"].
template get<double>());
655 const long resident = timings[
"batch_cache_hits"].template get<long>();
656 const long fetched = timings[
"batch_cache_misses"].template get<long>();
657 const long ahead = timings[
"batch_prefetch_hits"].template get<long>();
658 if (resident + fetched + ahead > 0)
659 printf(
" operand batches resident/ahead/fetched %6ld /%6ld /%6ld\n",
660 resident, ahead, fetched);
690 std::string
info()
const {
return "K";}
723 print(
"set macrotaskinfo to");
780 vecfuncT K_macrotask_efficient(
const vecfuncT& vket,
const double mul_tol = 0.0)
const;
783 vecfuncT K_macrotask_efficient_row(
const vecfuncT& vket,
const double mul_tol = 0.0)
const;
786 vecfuncT K_small_memory(
const vecfuncT& vket,
const double mul_tol = 0.0)
const;
789 vecfuncT K_large_memory(
const vecfuncT& vket,
const double mul_tol = 0.0)
const;
793 const vecfuncT& vket, std::shared_ptr<real_convolution_3d> poisson,
794 const bool symmetric,
const double mul_tol = 0.0);
883 std::shared_ptr<Future<batch_bytesT>>
fut;
943 const std::size_t per_node = std::size_t(1) << (2 *
NDIM);
944 return std::max<std::size_t>(1, (1u << 20) / (per_node *
k *
k *
k *
sizeof(T) + 1));
990 if (slot->valid and slot->key == record) {
991 vecfuncT data = cloud.template deserialize_batch_p2p<T, NDIM>(
992 world, *slot->fut, record,
false);
1025 const std::string
policy)
const override {
1042 const std::vector<Batch_1D> batches =
1045 for (
long i = 0; i < long(batches.size()); ++i) {
1046 for (
long j = 0; j <= i; ++j) {
1057 for (
auto i = partition1.begin(); i != partition1.end(); ++i) {
1059 for (
auto j = i; j != partition1.end(); ++j) {
1060 Batch batch(i->first.input[0], j->first.input[0],
_);
1062 result.push_back(std::make_pair(
batch,priority));
1065 for (
auto j = partition2.begin(); j != partition2.end(); ++j) {
1066 Batch batch(i->first.input[0], j->first.input[0],
_);
1068 result.push_back(std::make_pair(
batch,priority));
1080 long nrow =
batch.input[0].size();
1081 long ncol =
batch.input[1].size();
1082 return double(nrow * ncol);
1090 const int accumulation_mode = 2,
const bool cost_aware =
true,
1091 const long batch_salt = 0,
const bool bra_shares_ket =
true,
1092 const bool vf_shares_ket =
true)
1100 name=
"MacroTaskExchangeSimple";
1128 const long nsubworld) {
1135 std::map<long,long> column_index, row_index;
1136 for (
const auto& [task_batch, priority] : partition) {
1138 "owner-pinned exchange expects two-dimensional task batches");
1139 column_index[task_batch.input[0].begin] = 0;
1140 row_index[task_batch.input[1].begin] = 0;
1143 for (
auto& [begin, index] : column_index) index = next++;
1145 for (
auto& [begin, index] : row_index) index = next++;
1148 long(row_index.size()), nsubworld);
1149 for (
const auto& [task_batch, priority] : partition) {
1150 const long c = column_index[task_batch.input[0].begin];
1151 const long r = row_index[task_batch.input[1].begin];
1152 auto it = assignment.find({
c, r});
1153 owner_map_[{task_batch.input[0].begin, task_batch.input[1].begin}] =
1154 (it != assignment.end()) ? it->second : (
c % nsubworld);
1164 const std::vector<Batch_1D>
split =
1166 const long M = long(
split.size());
1167 std::map<long,long> begin_to_index;
1168 for (
long k = 0;
k < M; ++
k) begin_to_index[
split[
k].begin] =
k;
1176 const std::size_t ntask = std::size_t(M) * std::size_t(M + 1) / 2;
1180 const std::map<std::pair<long,long>,
long> assignment =
1184 for (
const auto& [task_batch, priority] : partition) {
1186 "owner-pinned exchange expects two-dimensional task batches");
1187 const long column_begin = task_batch.input[0].begin;
1188 const long row_begin = task_batch.input[1].begin;
1189 auto ic = begin_to_index.find(column_begin);
1190 auto jr = begin_to_index.find(row_begin);
1192 "owner-pinned exchange: a task batch is not one of the split batches");
1194 const long i = std::max(ic->second, jr->second);
1195 const long j = std::min(ic->second, jr->second);
1196 auto it = assignment.find({i, j});
1198 (it != assignment.end()) ? it->second : (i % nsubworld);
1210 "owner-pinned exchange expects two-dimensional task batches");
1212 return (it !=
owner_map_.end()) ? it->second : -1;
1217 typedef std::tuple<const std::vector<Function<T, NDIM>>&,
1218 const std::vector<Function<T, NDIM>>&,
1226 std::size_t n = std::get<0>(argtuple).size();
1249 const long nsubworld) {
1256 "owner-pinned exchange needs one subworld per rank");
1257 const vecfuncT& vf = std::get<0>(argtuple);
1270 std::vector<std::pair<long, vecfuncT>> owned;
1273 auto stage = [&](
const vecfuncT&
v,
const int dim,
const std::vector<Batch_1D>&
split) {
1274 for (
long k = 0;
k < long(
split.size()); ++
k) {
1278 if (
k % nsubworld ==
world.rank()) {
1280 for (
long i = r.
begin; i < r.
end; ++i)
1281 local[i - r.
begin] =
copy(subworld,
v[i],
false);
1282 owned.emplace_back(record, std::move(local));
1290 "exchange: bra and ket are a paired set and must have equal length");
1295 const std::vector<Batch_1D>
split =
1312 for (
auto& [record,
batch] : owned)
1334 const bool has_next)
const {
1343 if (not has_next or
cloud_ptr ==
nullptr)
return;
1350 const std::vector<Batch_1D> candidates =
1351 symmetric ? std::vector<Batch_1D>{next_col, next_row}
1352 : std::vector<Batch_1D>{next_row};
1353 for (
const auto& r : candidates) {
1423 std::vector<long> idx;
1424 auto add_range = [&](
const Batch_1D&
b) {
1425 const long s =
b.is_full_size() ? 0 :
b.begin;
1426 const long e =
b.is_full_size() ? long(
nresult) :
b.end;
1427 for (
long i = s; i <
e and i < long(
nresult); ++i) idx.push_back(i);
1429 if (
batch.input.empty()) {
1430 for (
long i = 0; i < long(
nresult); ++i) idx.push_back(i);
1433 add_range(
batch.input[0]);
1435 add_range(
batch.input[1]);
1436 std::sort(idx.begin(), idx.end());
1437 idx.erase(std::unique(idx.begin(), idx.end()), idx.end());
1443 const long wid = long(subworld.
id());
1453 rs_sub.reserve(touched.size());
1454 kf_sub.reserve(touched.size());
1455 for (
long i : touched) { rs_sub.push_back(result_subworld[i]); kf_sub.push_back(
Kf_local_[i]); }
1456 if (rs_sub.empty())
return;
1460 gaxpy(1.0, kf_sub, 1.0, rs_sub,
false);
1471 const long wid = long(nodeworld.
id());
1475 for (
long i = 0; i <
nresult; ++i)
1487 if (not nodeworld)
return;
1502 if (universe_result.empty())
return;
1503 World& universe = universe_result.front().world();
1521 std::vector<Function<T, NDIM>>
1532 bool diagonal_block =
batch.input[0] ==
batch.input[1];
1533 auto& bra_range =
batch.input[1];
1534 auto& vf_range =
batch.input[0];
1536 if (vf_range.is_full_size()) vf_range.end = vf_batch.size();
1537 if (bra_range.is_full_size()) bra_range.end = bra_batch.size();
1542 const double tile_wall_start =
wall_time();
1548 prof_.subworld_id =
static_cast<unsigned long>(
world.id());
1552 prof_.diagonal = diagonal_block;
1553 prof_.row_begin = bra_range.begin;
prof_.row_end = bra_range.end;
1554 prof_.col_begin = vf_range.begin;
prof_.col_end = vf_range.end;
1555 prof_.wall_start = tile_wall_start;
1561 vecfuncT bra_owned, vf_owned, ket_row_owned, ket_column_owned;
1562 const vecfuncT* bra_work = &bra_batch;
1563 const vecfuncT* vf_work = &vf_batch;
1580 ket_column_owned = diagonal_block
1595 bra_work = &bra_owned;
1596 vf_work = &vf_owned;
1599 const double compute_wall_start =
wall_time();
1601 if (
profiling)
prof_.wait_for_data_wall = compute_wall_start - tile_wall_start;
1611 for (
int i = vf_range.begin; i < vf_range.end; ++i){
1612 Kf[i] += resultcolumn[i - vf_range.begin];}
1614 }
else if (
symmetric and not diagonal_block) {
1616 : vf_range.copy_batch(vket);
1619 *bra_work, *vf_work);
1621 for (
int i = bra_range.begin; i < bra_range.end; ++i){
1622 Kf[i] += resultcolumn[i - bra_range.begin];}
1623 for (
int i = vf_range.begin; i < vf_range.end; ++i){
1624 Kf[i] += resultrow[i - vf_range.begin];}
1627 *bra_work, *vf_work);
1628 for (
int i = vf_range.begin; i < vf_range.end; ++i)
1629 Kf[i] += resultcolumn[i - vf_range.begin];
1673 "symmetric diagonal tile: ket/bra batch size mismatch");
1675 "symmetric diagonal tile: vf/bra batch size mismatch");
1677 const long n = long(vf_batch.size());
1683 auto tick = [&](
double& acc,
const double t0) {
if (prof_on) acc +=
wall_time() - t0; };
1685 for (
long i = 0; i < n; ++i) {
1687 double w0 = prof_on ?
wall_time() : 0.0;
1688 const vecfuncT vf_subset(vf_batch.begin(), vf_batch.begin() + i + 1);
1690 tick(
prof_.mul1_wall, w0);
1693 tick(
prof_.truncate_wall, w0);
1699 psif =
apply(subworld, *poisson.get(), psif);
1700 tick(
prof_.apply_wall, w0);
1703 tick(
prof_.truncate_wall, w0);
1711 double w1 = prof_on ?
wall_time() : 0.0;
1713 tick(
prof_.mul2_wall, w1);
1715 for (
long j = 0; j <= i; ++j)
update[j] += row_contrib[j];
1716 for (
long j = 0; j < i; ++j) {
1719 tick(
prof_.mul2_wall, w1);
1721 update[i] += mirrored[0];
1728 return resultcolumn;
1753 std::pair<vecfuncT, vecfuncT> compute_offdiagonal_batch_in_symmetric_matrix(
World& subworld,
1781 name=
"MacroTaskExchangeRow";
1785 typedef std::tuple<const std::vector<Function<T, NDIM>>&,
1786 const std::vector<Function<T, NDIM>>&,
1794 std::size_t n = std::get<0>(argtuple).size();
1803 std::vector<Function<T, NDIM>>
1807 std::vector<Function<T,NDIM>> result;
1813 MADNESS_EXCEPTION(
"unknown algorithm in Exchange::MacroTaskExchangeRow::operator()",1);
1818 std::vector<Function<T,NDIM>>
1833 MADNESS_CHECK_THROW(vket.size()==1,
"out-of-bounds error in Exchange::MacroTaskExchangeRow::operator()");
1834 size_t min_tile = 10;
1835 size_t ntile = std::min(
mo_bra.size(), min_tile);
1837 for (
size_t ilo=0; ilo<
mo_bra.size(); ilo+=ntile){
1839 size_t iend = std::min(ilo+ntile,
mo_bra.size());
1847 tmp_psif =
apply(
world, *poisson.get(), tmp_psif);
1866 std::vector<Function<T,NDIM>>
1872 double total_execution_time=0.0;
1873 double total_fetch_time=0.0;
1874 double total_fetch_spawn_time=0.0;
1882 std::shared_ptr<World> fetching_world(
new World(comm.
Clone()));
1883 std::shared_ptr<World> executing_world(
new World(comm.
Clone()));
1885 print(
"time to create two worlds:",cpu1-cpu0,
"seconds");
1886 print(
"executing_world.id()",executing_world->id(),
"fetching_world.id()",fetching_world->id(),
"in MacroTaskExchangeRow");
1897 MADNESS_CHECK_THROW(vket.size()==1,
"out-of-bounds error in Exchange::MacroTaskExchangeRow::operator()");
1898 size_t min_tile = 10;
1899 size_t ntile = std::min(
mo_bra.size(), min_tile);
1909 auto fetch_data = [&](
World&
world,
const Tile& tile) {
1911 "bra and ket size mismatch in Exchange::MacroTaskExchangeRow::execute()");
1913 std::size_t sz=tile.iend-tile.ilo;
1916 for (
size_t i=tile.ilo; i<tile.iend; ++i) {
1918 subworld_bra[i-tile.ilo]=
f;
1921 return std::make_pair(subworld_bra,subworld_ket);
1927 "bra and ket size mismatch in Exchange::MacroTaskExchangeRow::execute()");
1929 auto world_id=
world.id();
1930 auto phi_id=phi.world().id();
1931 auto bra_id=
mo_bra.front().world().id();
1932 auto ket_id=
mo_ket.front().world().id();
1933 std::string msg=
"world mismatch in Exchange::MacroTaskExchangeRow::execute(): ";
1934 msg+=
"world.id()="+std::to_string(world_id)+
", ";
1935 msg+=
"phi.world().id()="+std::to_string(phi_id)+
", ";
1936 msg+=
"bra.world().id()="+std::to_string(bra_id)+
", ";
1937 msg+=
"ket.world().id()="+std::to_string(ket_id);
1938 if (not (world_id==phi_id && world_id==bra_id && world_id==ket_id)) {
1941 MADNESS_CHECK_THROW(world_id==phi_id && world_id==bra_id && world_id==ket_id,msg.c_str());
1950 tmp_psif =
apply(
world, *poisson.get(), tmp_psif);
1960 return tmp_Kf.truncate();
1964 std::vector<Tile> tiles;
1965 for (
size_t ilo=0; ilo<
mo_bra.size(); ilo+=ntile) {
1966 tiles.push_back(Tile{ilo,std::min(ilo+ntile,
mo_bra.size())});
1972 for (
size_t itile=0; itile<tiles.size(); ++itile) {
1973 Tile& tile = tiles[itile];
1977 print(
"fetching tile",tile.ilo,
"into world",executing_world->id());
1978 std::tie(tmp_mo_bra1,tmp_mo_ket1)=fetch_data(*executing_world,tiles[itile]);
1979 fetching_world->gop.set_forbid_fence(
false);
1981 executing_world->gop.fence();
1983 total_fetch_time += (t1 - t0);
1984 total_fetch_spawn_time += (t2 - t0);
1988 fetching_world->gop.set_forbid_fence(
true);
1989 if (itile<tiles.size()-1) {
1991 print(
"fetching tile",tiles[itile+1].ilo,
"into world",fetching_world->id(),
" at time ",
wall_time());
1992 std::tie(tmp_mo_bra2,tmp_mo_ket2)=fetch_data(*fetching_world,tiles[itile+1]);
1994 fetching_world->gop.set_forbid_fence(
false);
1999 total_fetch_time += (t1 - t0);
2000 total_fetch_spawn_time += (t2 - t0);
2002 print(
"executing tile",tile.ilo,
"in world",executing_world->id());
2004 Kf[0]+=execute(*executing_world,poisson1,phi1,tmp_mo_bra1,tmp_mo_ket1);
2006 print(
"time to execute tile",tile.ilo,
"in world",executing_world->id(),dpu1-dpu0,
"seconds");
2007 total_execution_time += dpu1-dpu0;
2009 fetching_world->gop.fence();
2012 std::swap(poisson1,poisson2);
2013 std::swap(phi1,phi2);
2014 std::swap(tmp_mo_bra2,tmp_mo_bra1);
2015 std::swap(tmp_mo_ket2,tmp_mo_ket1);
2016 std::swap(executing_world,fetching_world);
2021 fetching_world->gop.fence();
2022 executing_world->gop.fence();
2024 print(
"overall time: ",cpu2-cpu0,
"seconds");
2025 print(
"total execution time:",total_execution_time,
"seconds");
2026 print(
"total fetch time:",total_fetch_time,
"seconds");
2027 print(
"total fetch spawn time:",total_fetch_spawn_time,
"seconds");
Operators for the molecular HF and DFT code.
static const double small
Definition binaryop.cc:117
Definition test_ar.cc:118
Definition test_ar.cc:141
Definition test_ar.cc:170
Wrapper around MPI_Comm. Has a shallow copy constructor; use Create(Get_group()) for deep copy.
Definition safempi.h:645
Intracomm Clone() const
Definition safempi.h:844
Definition macrotaskpartitioner.h:55
long size() const
Definition macrotaskpartitioner.h:71
long end
first and first past last index [begin,end)
Definition macrotaskpartitioner.h:59
long begin
Definition macrotaskpartitioner.h:59
a batch consists of a 2D-input batch and a 1D-output batch: K-batch <- (I-batch, J-batch)
Definition macrotaskpartitioner.h:124
std::vector< Batch_1D > input
Definition macrotaskpartitioner.h:127
cloud class
Definition cloud.h:338
Future< batch_bytesT > request_batch_bytes_async(const keyT record) const
start fetching record from its owner; the trigger is in flight on return
Definition cloud.h:1048
ProcessID batch_owner(const keyT record) const
the owner of a batch record; a pmap lookup, no communication
Definition cloud.h:1008
void register_batch_owner(const keyT record, const ProcessID owner)
Register the owner of a batch record; local map insert, no communication.
Definition cloud.h:783
keyT store_batch(madness::World &world, const std::vector< Function< T, NDIM > > &batch, const ProcessID owner, const keyT record, const bool fence=true)
Store a batch of functions as one owner-pinned record.
Definition cloud.h:797
Bounded cache of fetched exchange batches, keyed by cloud record key.
Definition exchangeoperator.h:443
std::list< Slot > slots_
Definition exchangeoperator.h:491
void set_transient_capacity(const std::size_t c)
how many non-owned entries may be resident; at least one is always allowed
Definition exchangeoperator.h:446
std::size_t size() const
Definition exchangeoperator.h:481
std::size_t transient_capacity_
Definition exchangeoperator.h:492
void clear()
drop every entry; the capacity setting survives
Definition exchangeoperator.h:479
std::size_t transient_capacity() const
Definition exchangeoperator.h:447
bool contains(const keyT &key) const
Definition exchangeoperator.h:449
std::size_t n_transient() const
Definition exchangeoperator.h:483
const dataT & insert(const keyT &key, dataT &&data, const bool pinned)
Insert as most-recently-used. pinned marks a batch this rank owns.
Definition exchangeoperator.h:466
const dataT * find(const keyT &key)
Definition exchangeoperator.h:455
custom partitioning for the exchange operator in exchangeoperator.h
Definition exchangeoperator.h:1770
MacroTaskPartitionerRow()
Definition exchangeoperator.h:1772
resultT allocator(World &world, const argtupleT &argtuple) const
Definition exchangeoperator.h:1793
std::vector< Function< T, NDIM > > row_fetch_compute(const std::vector< Function< T, NDIM > > &vket, const std::vector< Function< T, NDIM > > &mo_bra, const std::vector< Function< T, NDIM > > &mo_ket)
Definition exchangeoperator.h:1867
std::vector< Function< T, NDIM > > resultT
Definition exchangeoperator.h:1789
std::vector< Function< T, NDIM > > operator()(const std::vector< Function< T, NDIM > > &vket, const std::vector< Function< T, NDIM > > &mo_bra, const std::vector< Function< T, NDIM > > &mo_ket)
compute exchange row-wise for a fixed orbital phi_i of vket
Definition exchangeoperator.h:1804
double lo
Definition exchangeoperator.h:1764
bool symmetric
Definition exchangeoperator.h:1766
long nresult
Definition exchangeoperator.h:1763
std::tuple< const std::vector< Function< T, NDIM > > &, const std::vector< Function< T, NDIM > > &, const std::vector< Function< T, NDIM > > & > argtupleT
Definition exchangeoperator.h:1787
std::vector< Function< T, NDIM > > row(const std::vector< Function< T, NDIM > > &vket, const std::vector< Function< T, NDIM > > &mo_bra, const std::vector< Function< T, NDIM > > &mo_ket)
Definition exchangeoperator.h:1819
Algorithm algorithm_
Definition exchangeoperator.h:1767
MacroTaskExchangeRow(const long nresult, const double lo, const double mul_tol, const Algorithm algorithm)
Definition exchangeoperator.h:1778
double mul_tol
Definition exchangeoperator.h:1765
custom partitioning for the exchange operator in exchangeoperator.h
Definition exchangeoperator.h:1009
double compute_priority(const Batch &batch) const override
compute the priority of this task for non-dumb scheduling
Definition exchangeoperator.h:1078
MacroTaskPartitionerExchange(const bool symmetric, const bool owner_pinned=false, const long granularity_level=1)
Definition exchangeoperator.h:1011
bool symmetric
Definition exchangeoperator.h:1018
long granularity_level
Definition exchangeoperator.h:1022
bool owner_pinned
Definition exchangeoperator.h:1021
partitionT do_partitioning(const std::size_t &vsize1, const std::size_t &vsize2, const std::string policy) const override
override this if you want your own partitioning
Definition exchangeoperator.h:1024
void ensure_cache_world(World &world) const
drop cached batches when the subworld changes; they belong to the old one
Definition exchangeoperator.h:966
static std::shared_ptr< FinalizeReducer< T, NDIM > > node_reducer_
Definition exchangeoperator.h:927
vecfuncT compute_diagonal_batch_in_symmetric_matrix(World &subworld, const vecfuncT &ket_batch, const vecfuncT &bra_batch, const vecfuncT &vf_batch) const
compute a batch of the exchange matrix, with identical ranges, exploiting the matrix symmetry
Definition exchangeoperator.h:1667
static long exchange_call_index()
Definition exchangeoperator.h:1111
static void clear_local_caches()
Definition exchangeoperator.h:933
void finalize_stage1(World &subworld, World *nodeworld)
reduce this subworld's accumulator into the node-shared one
Definition exchangeoperator.h:1486
static void commit_cost_reference()
make this call's measured costs the reference for the next one
Definition exchangeoperator.h:1115
static long batch_cache_hits()
Definition exchangeoperator.h:1105
void cleanup() override
Drop the cached batches while the subworld holding them is still alive.
Definition exchangeoperator.h:1374
static std::vector< double > cost_reference_
What each task cost last time, to place them better this time.
Definition exchangeoperator.h:896
bool wants_node_local_reduction() const
Definition exchangeoperator.h:1406
static ExchangeBatchLRU< long, vecfuncT > batch_cache_
Batches fetched from the cloud, reused across the tasks that run in one subworld.
Definition exchangeoperator.h:862
static FinalizeReducer< T, NDIM > & get_universe_reducer(World &world)
Definition exchangeoperator.h:949
void finalize_stage2(World &subworld, World *nodeworld, vecfuncT &universe_result)
drain into the universe result, from the node accumulator if there is one
Definition exchangeoperator.h:1499
static vecfuncT Kf_local_
Definition exchangeoperator.h:911
static std::vector< double > & cost_this_call()
Definition exchangeoperator.h:1110
bool profile_active() const
per-task profiling on for this task?
Definition exchangeoperator.h:1109
long nresult
Definition exchangeoperator.h:823
static std::map< long, long > batch_begin_to_index_
batch offset -> index
Definition exchangeoperator.h:898
double mul_tol
Definition exchangeoperator.h:825
static std::atomic< long > batch_cache_hits_
Definition exchangeoperator.h:864
vecfuncT compute_batch_in_asymmetric_matrix(World &subworld, const vecfuncT &ket_batch, const vecfuncT &bra_batch, const vecfuncT &vf_batch) const
compute a batch of the exchange matrix, with non-identical ranges
Definition exchangeoperator.h:1738
void ensure_node_accumulator(World &nodeworld) const
Collectively (re)build the node-shared accumulator in the node world.
Definition exchangeoperator.h:1470
int accumulation_mode_
Definition exchangeoperator.h:847
static FinalizeReducer< T, NDIM > & get_node_reducer(World &world)
Definition exchangeoperator.h:957
void prepare_owner_assignment(const MacroTaskPartitioner::partitionT &partition, const long nsubworld)
Assign every task to the rank that will own one of its two batches.
Definition exchangeoperator.h:1127
static bool Kf_local_initialized_
Definition exchangeoperator.h:912
static PrefetchSlot prefetch_next_
requested during this task
Definition exchangeoperator.h:886
long universe_rank_
this process's rank in the universe, to recognise the batches it owns
Definition exchangeoperator.h:854
bool symmetric
Definition exchangeoperator.h:826
static long universe_reducer_world_id_
Definition exchangeoperator.h:926
void sym_pipeline_advance(World &subworld, const vecfuncT &, const Batch_1D &next_col, const Batch_1D &next_row, const bool has_next) const
Request the batch the next task will have to fetch, before computing this one.
Definition exchangeoperator.h:1332
static bool finalize_stage1_done_
each drain happens once per rank, not once per task object
Definition exchangeoperator.h:930
bool accumulates_own_output() const override
Definition exchangeoperator.h:1400
static long batch_cache_misses()
Definition exchangeoperator.h:1106
static long prof_task_seq_
per-process task counter, for identity only
Definition exchangeoperator.h:904
void accumulate_locally(World &subworld, const vecfuncT &result_subworld) const
sum one tile's result into this subworld's accumulator
Definition exchangeoperator.h:1442
std::vector< long > touched_result_indices() const
The result entries this tile actually wrote.
Definition exchangeoperator.h:1422
ExchTaskProfile prof_
Definition exchangeoperator.h:903
bool cost_aware_
place tasks by their measured cost rather than by counting them
Definition exchangeoperator.h:849
static long exchange_call_index_
Definition exchangeoperator.h:899
std::vector< Function< T, NDIM > > operator()(const std::vector< Function< T, NDIM > > &vf_batch, const std::vector< Function< T, NDIM > > &bra_batch, const std::vector< Function< T, NDIM > > &vket)
Definition exchangeoperator.h:1522
bool owner_pinned
pin each task to a rank that owns one of its batches, over the owner-pinned split
Definition exchangeoperator.h:828
long granularity_level
Definition exchangeoperator.h:829
std::tuple< const std::vector< Function< T, NDIM > > &, const std::vector< Function< T, NDIM > > &, const std::vector< Function< T, NDIM > > & > argtupleT
Definition exchangeoperator.h:1219
bool handles_own_data_movement() const override
the owner-pinned path fetches its operand batches from the cloud itself
Definition exchangeoperator.h:1367
resultT allocator(World &world, const argtupleT &argtuple) const
Definition exchangeoperator.h:1225
bool bra_shares_ket_
Definition exchangeoperator.h:842
static void reset_batch_cache_counters()
Definition exchangeoperator.h:1116
static long Kf_node_world_id_
Definition exchangeoperator.h:916
std::pair< vecfuncT, vecfuncT > compute_offdiagonal_batch_in_symmetric_matrix(World &subworld, const vecfuncT &ket_rows, const vecfuncT &ket_columns, const vecfuncT &bra_batch, const vecfuncT &vf_batch) const
compute a batch of the exchange matrix, with non-identical ranges
Definition exchangeoperator.cc:353
static long Kf_local_world_id_
Definition exchangeoperator.h:913
std::vector< Function< T, NDIM > > resultT
Definition exchangeoperator.h:1221
static long batch_prefetch_hits()
Definition exchangeoperator.h:1107
double lo
Definition exchangeoperator.h:824
static std::atomic< long > batch_cache_misses_
Definition exchangeoperator.h:865
int bra_role() const
the record role carrying the bra, and the one carrying the vf over a shared split
Definition exchangeoperator.h:1203
static bool finalize_universe_done_
Definition exchangeoperator.h:931
static std::vector< double > cost_this_call_
rank-local, summed after the call
Definition exchangeoperator.h:897
std::map< std::pair< long, long >, long > owner_map_
Definition exchangeoperator.h:852
static bool Kf_node_initialized_
Definition exchangeoperator.h:915
const vecfuncT & fetch_batch(World &world, Cloud &cloud, const long record) const
Fetch one owner-pinned batch, from the local cache if it is resident.
Definition exchangeoperator.h:980
static long node_reducer_world_id_
Definition exchangeoperator.h:928
static long cost_matrix_dimension()
number of batches this application split into, which squares to the cost matrix
Definition exchangeoperator.h:1113
MacroTaskExchangeSimple(const long nresult, const double lo, const double mul_tol, const bool symmetric, const bool owner_pinned=false, const long granularity_level=1, const long universe_rank=0, const int accumulation_mode=2, const bool cost_aware=true, const long batch_salt=0, const bool bra_shares_ket=true, const bool vf_shares_ket=true)
Definition exchangeoperator.h:1087
void store_batches(World &world, World &subworld, Cloud &cloud, const argtupleT &argtuple, const long nsubworld)
Store the orbitals as owner-pinned batches, one record per batch.
Definition exchangeoperator.h:1248
bool vf_shares_ket_
Definition exchangeoperator.h:843
static PrefetchSlot prefetch_current_
promoted from the previous task
Definition exchangeoperator.h:885
static std::atomic< long > batch_prefetch_hits_
Definition exchangeoperator.h:866
static long cache_world_id_
Definition exchangeoperator.h:863
static vecfuncT Kf_node_
Definition exchangeoperator.h:914
static std::size_t finalize_chunk_entries()
entries per chunk in coalesced_gaxpy, sized so one message stays modest at any k
Definition exchangeoperator.h:941
int vf_role() const
Definition exchangeoperator.h:1204
static std::shared_ptr< FinalizeReducer< T, NDIM > > universe_reducer_
Receiving endpoints for the two drains, one per world they transfer within.
Definition exchangeoperator.h:925
long batch_salt_
Per-application salt for the batch record keys, from exchange_batch_salt.
Definition exchangeoperator.h:837
long owner_hint(const Batch &task_batch, const long nsubworld) const override
Definition exchangeoperator.h:1207
Definition exchangeoperator.h:601
long batch_granularity_
Definition exchangeoperator.h:811
static std::atomic< long > mul1_timer
timing
Definition exchangeoperator.h:607
Exchange< T, NDIM >::ExchangeAlgorithm Algorithm
Definition exchangeoperator.h:665
bool printtimings() const
Definition exchangeoperator.h:798
MacroTaskInfo macro_task_info
Definition exchangeoperator.h:667
ExchangeImpl & symmetric(const bool flag)
Definition exchangeoperator.h:710
bool cost_aware_assign_
Definition exchangeoperator.h:817
ExchangeImpl & set_printlevel(const long &level)
Definition exchangeoperator.h:734
nlohmann::json statistics
statistics of the Cloud (timings, memory) and of the parameters of this run
Definition exchangeoperator.h:819
static double elapsed_time
Definition exchangeoperator.h:608
void print_timer(World &world) const
Definition exchangeoperator.h:646
ExchangeImpl & set_batch_granularity(const long level)
Definition exchangeoperator.h:739
World & get_world() const
Definition exchangeoperator.h:758
ExchangeImpl & set_macro_task_info(const std::vector< std::string > &info)
Definition exchangeoperator.h:720
double thresh
Definition exchangeoperator.h:806
nlohmann::json get_statistics() const
Definition exchangeoperator.h:760
ExchangeImpl & set_macro_task_info(const MacroTaskInfo &info)
Definition exchangeoperator.h:715
long printlevel
Definition exchangeoperator.h:807
static void reset_timer()
Definition exchangeoperator.h:610
vecfuncT mo_bra
is the exchange matrix symmetric? K phi_i = \sum_k \phi_k \int \phi_k \phi_i
Definition exchangeoperator.h:804
double lo
Definition exchangeoperator.h:805
World & world
Definition exchangeoperator.h:801
std::shared_ptr< MacroTaskQ > taskq
Definition exchangeoperator.h:802
Algorithm algorithm_
Definition exchangeoperator.h:666
int accumulation_mode_
Definition exchangeoperator.h:814
Function< T, NDIM > functionT
Definition exchangeoperator.h:602
nlohmann::json gather_statistics() const
return some statistics about the current settings
Definition exchangeoperator.h:763
bool is_symmetric() const
Definition exchangeoperator.h:703
ExchangeImpl & set_taskq(std::shared_ptr< MacroTaskQ > taskq1)
Definition exchangeoperator.h:705
std::vector< functionT > vecfuncT
Definition exchangeoperator.h:603
double mul_tol
Definition exchangeoperator.h:808
static vecfuncT compute_K_tile(World &world, const vecfuncT &mo_bra, const vecfuncT &mo_ket, const vecfuncT &vket, std::shared_ptr< real_convolution_3d > poisson, const bool symmetric, const double mul_tol=0.0)
computing the upper triangle of the double sum (over vket and the K orbitals)
Definition exchangeoperator.cc:273
ExchangeImpl & set_cost_aware_assignment(const bool flag)
Definition exchangeoperator.h:745
ExchangeImpl & set_algorithm(const Algorithm &alg)
Definition exchangeoperator.h:729
bool printprogress() const
Definition exchangeoperator.h:797
static std::atomic< long > apply_timer
Definition exchangeoperator.h:605
ExchangeImpl & set_accumulation_mode(const int mode)
Definition exchangeoperator.h:750
vecfuncT mo_ket
MOs for bra and ket.
Definition exchangeoperator.h:804
ExchangeImpl(World &world, const double lo, const double thresh)
default ctor
Definition exchangeoperator.h:670
std::string info() const
Definition exchangeoperator.h:690
nlohmann::json gather_timings(World &world) const
Definition exchangeoperator.h:619
bool printtimings_detail() const
Definition exchangeoperator.h:799
bool symmetric_
Definition exchangeoperator.h:803
bool printdebug() const
Definition exchangeoperator.h:796
std::shared_ptr< MacroTaskQ > get_taskq() const
Definition exchangeoperator.h:756
static auto set_poisson(World &world, const double lo, const double econv=FunctionDefaults< 3 >::get_thresh())
Definition exchangeoperator.h:692
static std::atomic< long > mul2_timer
Definition exchangeoperator.h:606
void set_bra_and_ket(const vecfuncT &bra, const vecfuncT &ket)
set the bra and ket orbital spaces, and the occupation
Definition exchangeoperator.h:685
Exchange(World &world, const double lo, const double thresh=FunctionDefaults< NDIM >::get_thresh())
ctor
Definition exchangeoperator.cc:418
static std::string to_string(const ExchangeAlgorithm alg)
Definition SCFOperators.h:145
ExchangeAlgorithm
Definition SCFOperators.h:118
@ fetch_compute
Definition SCFOperators.h:119
@ multiworld_efficient_row
Definition SCFOperators.h:119
Function< T, NDIM > operator()(const Function< T, NDIM > &ket) const
Definition SCFOperators.h:211
std::vector< functionT > vecfuncT
Definition SCFOperators.h:112
Receiving end of the exchange finalize, living in the world the transfer rides on.
Definition exchangeoperator.h:516
T beta_
Definition exchangeoperator.h:547
FinalizeReducer(World &world)
Definition exchangeoperator.h:521
std::vector< implT * > dests_
Definition exchangeoperator.h:546
FinalizeNodeRec< T, NDIM > recT
Definition exchangeoperator.h:519
FunctionImpl< T, NDIM > implT
Definition exchangeoperator.h:518
void accumulate_chunk(const std::vector< recT > &recs)
Definition exchangeoperator.h:532
void set_targets(std::vector< implT * > dests, T beta)
point at the destinations for the next drain; local, called by every rank
Definition exchangeoperator.h:527
static int get_k()
Returns the default wavelet order.
Definition funcdefaults.h:170
static std::shared_ptr< WorldDCPmapInterface< Key< NDIM > > > make_default_pmap(World &world)
Makes a default process map for the given world.
Definition mraimpl.h:3840
static const double & get_thresh()
Returns the default threshold.
Definition funcdefaults.h:183
static void set_default_pmap(World &world)
Definition mraimpl.h:3847
static std::shared_ptr< WorldDCPmapInterface< Key< NDIM > > > & get_pmap()
Returns the default process map that was last initialized via set_default_pmap().
Definition funcdefaults.h:415
static void set_pmap(const std::shared_ptr< WorldDCPmapInterface< Key< NDIM > > > &value)
Sets the default process map (does not redistribute existing functions).
Definition funcdefaults.h:446
FunctionFactory implements the named-parameter idiom for Function.
Definition function_factory.h:86
FunctionImpl holds all Function state to facilitate shallow copy semantics.
Definition funcimpl.h:1043
FunctionNode holds the coefficients, etc., at each node of the 2^NDIM-tree.
Definition funcimpl.h:136
A multiresolution adaptive numerical function.
Definition mra.h:141
Key is the index for a node of the 2^NDIM-tree.
Definition key.h:71
Cloud * cloud_ptr
Definition macrotaskq.h:1639
MacroTaskOperationBase()
Definition macrotaskq.h:1642
Batch batch
Definition macrotaskq.h:1635
World * subworld_ptr
Definition macrotaskq.h:1636
std::shared_ptr< MacroTaskPartitioner > partitioner
Definition macrotaskq.h:1641
std::string name
Definition macrotaskq.h:1640
std::string policy
how to partition the batches
Definition macrotaskpartitioner.h:190
std::size_t nsubworld
number of worlds (try to have enough batches for all worlds)
Definition macrotaskpartitioner.h:189
std::list< std::pair< Batch, double > > partitionT
Definition macrotaskpartitioner.h:186
friend class Batch
Definition macrotaskpartitioner.h:183
MacroTaskPartitioner()
Definition macrotaskpartitioner.h:193
partitionT do_1d_partition(const std::size_t vsize, const std::string policy) const
Definition macrotaskpartitioner.h:251
std::size_t max_batch_size
maximum batch size (for memory management)
Definition macrotaskpartitioner.h:188
The Nemo class.
Definition nemo.h:377
void fence(bool debug=false)
Synchronizes all processes in communicator AND globally ensures no pending AM or tasks.
Definition worldgop.cc:177
WorldObject(const WorldObject &other)
Definition world_object.h:500
void process_pending()
Definition world_object.h:787
detail::task_result_type< memfnT >::futureT task(ProcessID dest, memfnT memfn, const TaskAttributes &attr=TaskAttributes()) const
Sends task to derived class method returnT (this->*memfn)().
Definition world_object.h:1132
A parallel world class.
Definition world.h:134
unsigned long id() const
Definition world.h:324
WorldGopInterface & gop
Global operations.
Definition world.h:216
Declares the Cloud class for storing data and transfering them between worlds.
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition derivatives.cc:72
std::vector< Spinor > truncate(std::vector< Spinor > arg)
Definition dirac-hatom.cc:503
Fcwf apply(World &world, real_convolution_3d &op, const Fcwf &psi)
Definition fcwf.cc:317
Fcwf mul_sparse(const Fcwf &psi, const madness::real_function_3d &phi, double tol)
Definition fcwf.cc:566
Fcwf copy(Fcwf psi)
Definition fcwf.cc:374
const double beta
Definition gygi_soltion.cc:62
static const double v
Definition hatom_sf_dirac.cc:20
Declares the macrotaskq and MacroTaskBase classes.
General header file for using MADNESS.
#define MADNESS_CHECK(condition)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:182
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_CHECK_THROW(condition, msg)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:207
Function< double, 3 > functionT
Definition mcpfit.cc:50
vector< functionT > vecfuncT
Definition mcpfit.cc:51
void print(const tensorT &t)
Definition mcpfit.cc:140
Namespace for all elements and tools of MADNESS.
Definition DFConvergence.h:9
std::vector< std::pair< Batch_1D, Batch_1D > > exchange_row_owner_grid(const std::size_t ncolumn, const std::size_t nrow, const long nsubworld)
The asymmetric task grid: every (column, row) pair over two independent splits.
Definition exchangeoperator.h:79
long exchange_sym_owner_nbatch(const std::size_t n, const long nsubworld, const long granularity_level)
Number of batches M for the owner-pinned symmetric exchange algorithm.
Definition exchangeoperator.h:31
double get_rss_usage_in_GB()
Definition ranks_and_hosts.cpp:18
static double process_cpu_time()
Returns the CPU time consumed by this process, in seconds.
Definition timers.h:152
void coalesced_gaxpy(World &transport_world, FinalizeReducer< T, NDIM > &reducer, std::vector< Function< T, NDIM > > &dest_vec, std::vector< Function< T, NDIM > > &src_vec, const T beta, const std::size_t chunk_entries)
Accumulate src_vec into dest_vec in bulk, one message per destination rank per chunk.
Definition exchangeoperator.h:562
std::vector< Batch_1D > exchange_sym_owner_split(const std::size_t n, const long nsubworld, const long granularity_level)
Split a vector of length n into M = exchange_sym_owner_nbatch(...) contiguous batches.
Definition exchangeoperator.h:45
std::shared_ptr< WorldDCPmapInterface< Key< NDIM > > > FunctionDefaults< NDIM >::pmap
Definition mraimpl.h:3898
Function< TENSOR_RESULT_TYPE(T, R), NDIM > dot_sparse(World &world, const std::vector< Function< T, NDIM > > &a, const std::vector< Function< R, NDIM > > &b, double tol, bool fence=true, bool do_make_redundant=true)
Multiplies and sums two vectors of functions r = \sum_i a[i] * b[i].
Definition vmra.h:1884
std::map< std::pair< long, long >, long > exchange_sym_cost_aware_assign(const long n, const long M, const std::vector< double > &cost)
Cost-aware owner assignment for the symmetric algorithm's triangular task matrix.
Definition exchangeoperator.h:242
long exchange_batch_record_key(const long salt, const int dim, const Batch_1D &r)
Deterministic cloud record key for one stored batch: (salt, dimension, range).
Definition exchangeoperator.h:424
int FunctionDefaults< NDIM >::k
Definition mraimpl.h:3878
Function< TENSOR_RESULT_TYPE(T, R), NDIM > dot(World &world, const std::vector< Function< T, NDIM > > &a, const std::vector< Function< R, NDIM > > &b, bool fence=true, bool do_make_redundant=true)
Multiplies and sums two vectors of functions r = \sum_i a[i] * b[i]; see dot_sparse for screening.
Definition vmra.h:1897
std::vector< std::shared_ptr< FunctionImpl< T, NDIM > > > get_impl(const std::vector< Function< T, NDIM > > &v)
Definition vmra.h:781
TreeState
Definition funcdefaults.h:60
@ reconstructed
s coeffs at the leaves only
Definition funcdefaults.h:61
@ compressed
d coeffs in internal nodes, s and d coeffs at the root, empty leaves may be present
Definition funcdefaults.h:62
long exchange_sym_tri(const long i, const long j)
Triangular index of a batch pair, collapsing (i,j) and (j,i): a*(a+1)/2 + b.
Definition exchangeoperator.h:124
void compress(World &world, const std::vector< Function< T, NDIM > > &v, bool fence=true)
Compress a vector of functions.
Definition vmra.h:150
void hash_combine(hashT &seed, const T &v)
Combine hash values.
Definition worldhash.h:261
std::map< std::pair< long, long >, long > exchange_row_owner_assign(const long ncolumn, const long nrow, const long nworker)
Owner of every task in the asymmetric grid: all tasks of a column go to one worker.
Definition exchangeoperator.h:112
static const Slice _(0,-1, 1)
bool exch_task_profile_enabled()
Is per-task exchange profiling on? Read once per process.
Definition exchangeoperator.h:323
long exchange_batch_salt(const std::vector< Function< T, NDIM > > &ket)
Per-invocation salt for the exchange batch record keys, from the ket identities.
Definition exchangeoperator.h:415
void exch_write_task_profile(const ExchTaskProfile &p)
Append one record to exch_taskprof.r<rank>.jsonl.
Definition exchangeoperator.h:344
std::vector< Batch_1D > exchange_row_owner_split(const std::size_t n, const long nsubworld)
Batch boundaries for the asymmetric row/column split: one batch per rank.
Definition exchangeoperator.h:67
@ TT_FULL
Definition gentensor.h:120
NDIM & f
Definition mra.h:2817
const Function< T, NDIM > & change_tree_state(const Function< T, NDIM > &f, const TreeState finalstate, bool fence=true)
change tree state of a function
Definition mra.h:3143
double wall_time()
Returns the wall time in seconds relative to an arbitrary origin.
Definition timers.cc:48
void print(const T &t, Ts &&... ts)
Print items to std::cout (items separated by spaces) and terminate with a new line.
Definition print.h:232
static SeparatedConvolution< double, 3 > * CoulombOperatorPtr(World &world, double lo, double eps, const std::array< LatticeRange, 3 > &lattice_ranges=FunctionDefaults< 3 >::get_bc().lattice_range(), int k=FunctionDefaults< 3 >::get_k())
Factory function generating separated kernel for convolution with 1/r in 3D.
Definition operator.h:2039
std::vector< Function< T, NDIM > > zero_functions_compressed(World &world, int n, bool fence=true)
Generates a vector of zero functions (compressed).
Definition vmra.h:450
bool exch_force_general_path()
Send a symmetric application down the general (bra, ket, vf) path? Read once per process.
Definition exchangeoperator.h:334
void exch_write_cost_matrix(const long call_index, const long k, const long M, const std::vector< double > &cost)
Write the measured per-task cost matrix of one application, for offline inspection.
Definition exchangeoperator.h:384
void load(Function< T, NDIM > &f, const std::string name)
Definition mra.h:3181
const FunctionCommonData< T, NDIM > * FunctionCommonData< T, NDIM >::data[MAXK]
Definition mraimpl.h:3875
bool exchange_same_operands(const std::vector< Function< T, NDIM > > &a, const std::vector< Function< T, NDIM > > &b)
Are these the same functions, so that one stored record can serve both operand roles?
Definition exchangeoperator.h:96
std::string name(const FuncType &type, const int ex=-1)
Definition ccpairfunction.h:28
madness::hashT hash_value(const std::array< T, N > &a)
Hash std::array with madness hash.
Definition array_addons.h:78
std::map< std::pair< long, long >, long > exchange_sym_round_robin_assign(const long n, const long M)
Round-robin owner assignment for the symmetric algorithm's triangular task matrix.
Definition exchangeoperator.h:148
ExchangeBatchDim
which of the three exchange operand vectors a stored batch belongs to
Definition exchangeoperator.h:401
@ EXCHANGE_BATCH_BRA
Definition exchangeoperator.h:401
@ EXCHANGE_BATCH_VF
Definition exchangeoperator.h:401
@ EXCHANGE_BATCH_KET
Definition exchangeoperator.h:401
Function< T, NDIM > copy(const Function< T, NDIM > &f, const std::shared_ptr< WorldDCPmapInterface< Key< NDIM > > > &pmap, bool fence=true)
Create a new copy of the function with different distribution and optional fence.
Definition mra.h:2239
void gaxpy(const double a, ScalarResult< T > &left, const double b, const T &right, const bool fence=true)
the result type of a macrotask must implement gaxpy
Definition macrotaskq.h:243
@ nemo
nemo's regularized orbitals F = psi/R
Definition Restart.h:60
static const double b
Definition nonlinschro.cc:119
static const double a
Definition nonlinschro.cc:118
double C(double a)
Additional factor when transforming gaussian exponent a from fourier to real space.
Definition relops.cc:25
static const double c
Definition relops.cc:10
One task's record for the exchange profiler.
Definition exchangeoperator.h:293
double mul1_wall
Definition exchangeoperator.h:309
double compute_cpu
Definition exchangeoperator.h:304
double mul2_wall
Definition exchangeoperator.h:309
double thresh
which protocol tier this task ran in
Definition exchangeoperator.h:298
long k
Definition exchangeoperator.h:299
long task_id
Definition exchangeoperator.h:294
double truncate_wall
Definition exchangeoperator.h:309
long col_end
Definition exchangeoperator.h:301
double apply_wall
Definition exchangeoperator.h:309
unsigned long subworld_id
Definition exchangeoperator.h:296
long row_begin
Definition exchangeoperator.h:301
bool waited
a cold fetch happened, so this task paid latency
Definition exchangeoperator.h:311
double wait_for_data_wall
task entry until its operands are in hand
Definition exchangeoperator.h:303
long col_begin
Definition exchangeoperator.h:301
void reset()
Definition exchangeoperator.h:314
double wall_end
Definition exchangeoperator.h:302
long universe_rank
keys the output file: one per process
Definition exchangeoperator.h:295
int subworld_nrank
Definition exchangeoperator.h:297
double compute_wall
Definition exchangeoperator.h:304
bool diagonal
Definition exchangeoperator.h:300
long row_end
Definition exchangeoperator.h:301
double wall_start
Definition exchangeoperator.h:302
int operand_source
worst of its fetches: 0 resident, 1 ahead, 2 cold
Definition exchangeoperator.h:310
void observe_fetch_tier(const int tier)
keep the worst source, since that is the one that set the task's wait
Definition exchangeoperator.h:316
double peak_rss_gb
Definition exchangeoperator.h:312
Definition exchangeoperator.h:490
dataT data
Definition exchangeoperator.h:490
bool pinned
Definition exchangeoperator.h:490
keyT key
Definition exchangeoperator.h:490
One batch requested ahead of the task that will read it.
Definition exchangeoperator.h:876
std::shared_ptr< Future< batch_bytesT > > fut
Definition exchangeoperator.h:883
long key
Definition exchangeoperator.h:878
bool valid
Definition exchangeoperator.h:877
One coefficient node in transit during the exchange finalize.
Definition exchangeoperator.h:501
std::size_t f
index into the destination function vector
Definition exchangeoperator.h:502
Key< NDIM > key
tree-node key
Definition exchangeoperator.h:503
void serialize(Archive &ar)
Definition exchangeoperator.h:506
FunctionNode< T, NDIM > node
the source node
Definition exchangeoperator.h:504
Definition macrotaskq.h:279
static MacroTaskInfo preset(const std::string name)
Definition macrotaskq.h:312
World & world
Memoized reference to the world to which this object belongs.
Definition world_object.h:348
class to temporarily redirect output to cout
Definition print.h:305
void split(const Range< ConcurrentHashMap< int, int >::iterator > &range)
Definition test_hashthreaded.cc:63
double cpu_time()
Definition test_list.cc:43
void e()
Definition test_sig.cc:75
std::complex< double > dataT
Definition testcomplexfunctionsolver.cc:10
vector_complex_function_3d update(World &world, const vector_complex_function_3d &psi, vector_complex_function_3d &vpsi, const tensor_real &e, int iter)
Definition testcosine.cc:210
constexpr std::size_t NDIM
Definition testgconv.cc:54
int ProcessID
Used to clearly identify process number/rank.
Definition worldtypes.h:43
Key< D > keyT
Definition writecoeff2.cc:11