MADNESS 0.10.1
lbdeux.h
Go to the documentation of this file.
1/*
2 This file is part of MADNESS.
3
4 Copyright (C) 2007,2010 Oak Ridge National Laboratory
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
20 For more information please contact:
21
22 Robert J. Harrison
23 Oak Ridge National Laboratory
24 One Bethel Valley Road
25 P.O. Box 2008, MS-6367
26
27 email: harrisonrj@ornl.gov
28 tel: 865-241-3937
29 fax: 865-572-0680
30
31 $Id$
32*/
33#ifndef MADNESS_MRA_IBDEUX_H__INCLUDED
34#define MADNESS_MRA_IBDEUX_H__INCLUDED
35
37#include <algorithm>
38#include <cmath>
39#include <iomanip>
40#include <iostream>
41#include <limits>
42#include <queue>
43#include <string>
44#include <unordered_map>
45
48
49#include <madness/mra/key.h>
51
52/// \file mra/lbdeux.h
53/// \brief Implements (2nd generation) static load/data balancing for functions
54/// \ingroup function
55
56namespace madness {
57
58 template<typename T, std::size_t NDIM>
59 class FunctionNode;
60
61 template<typename T, std::size_t NDIM>
62 class Function;
63
64 template <std::size_t NDIM>
65 class LBDeuxPmap : public WorldDCPmapInterface< Key<NDIM> > {
66 typedef Key<NDIM> keyT;
67 typedef std::pair<keyT,ProcessID> pairT;
68 // Only ever queried with find(), never iterated, so ordering buys nothing.
69 // Hash<keyT> returns the hash cached in Key.
70 typedef std::unordered_map<keyT,ProcessID,Hash<keyT>> mapT;
72 Level max_level = 0; ///< Deepest level of any entry in map
73 typedef typename mapT::const_iterator iteratorT;
74
75 public:
76 /// @param[in] v The owner of each subtree root. The root key must be present, keys must
77 /// be unique, and ranks must be in [0, nproc).
78 /// @param[in] nproc The number of processes in the World that uses this map
79 /// @throw MadnessException if v is not valid
80 LBDeuxPmap(const std::vector<pairT>& v, const ProcessID nproc) {
81 MADNESS_CHECK_THROW(!v.empty(), "LBDeuxPmap: empty assignment");
82 map.reserve(v.size());
83 for (const pairT& entry : v) {
84 MADNESS_CHECK_THROW(entry.second >= 0 && entry.second < nproc, "LBDeuxPmap: rank out of range");
85 const bool inserted = map.insert(entry).second;
86 MADNESS_CHECK_THROW(inserted, "LBDeuxPmap: duplicate key");
87 max_level = std::max(max_level, entry.first.level());
88 }
89 MADNESS_CHECK_THROW(map.count(keyT(0)) == 1, "LBDeuxPmap: missing root");
90 }
91
92 /// @return The owner of the nearest ancestor of key that is in the map, or of key itself
93 /// @throw MadnessException if no ancestor is in the map, which a valid key cannot cause
94 ProcessID owner(const keyT& key) const {
95 // No entry is deeper than max_level, so start the walk at the ancestor on that level.
96 keyT k = (key.level() > max_level) ? key.parent(key.level() - max_level) : key;
97 while (true) {
98 const iteratorT it = map.find(k);
99 if (it != map.end()) return it->second;
100 if (k.level() <= 0) {
101 madness::print("LBDeuxPmap: no owner for key", key);
102 MADNESS_EXCEPTION("LBDeuxPmap: lookup failed", 0);
103 }
104 k = k.parent();
105 }
106 }
107
108 void print() const {
109 madness::print("LBDeuxPmap");
110 }
111 };
112
113
114
115 template <std::size_t NDIM>
117 static const int nchild = (1<<NDIM);
121
122 double child_cost[nchild]; // Removed volatile since all parallel updates will be under mutex from active message and no unprotected reads
123 double my_cost;
126 bool deleted; ///< Set by deleter(). A retained node can have any cost, even a negative one.
127
129
130 /// Computes index of child key in this node using last bit of translations
131 int index(const keyT& key) {
132 int ind = 0;
133 for (std::size_t d=0; d<NDIM; ++d) ind += ((key.translation()[d])&0x1) << d;
134 return ind;
135 }
136
137 public:
139 : my_cost(0.0), total_cost(0.0), gotkids(false), deleted(false) {
140 nsummed = 0;
141 for (int i=0; i<nchild; ++i)
142 child_cost[i] = 0.0;
143 }
144
146 my_cost(other.my_cost), total_cost(other.total_cost), gotkids(other.gotkids), deleted(other.deleted)
147 {
148 nsummed = other.nsummed;
149 for (int i=0; i<nchild; ++i)
150 child_cost[i] = other.child_cost[i];
151 }
152
154 for (int i=0; i<nchild; ++i)
155 child_cost[i] = other.child_cost[i];
156 my_cost = other.my_cost;
157 total_cost = other.total_cost;
158 gotkids = other.gotkids;
159 deleted = other.deleted;
160 nsummed = other.nsummed;
161
162 return *this;
163 }
164
165 bool has_children() const {
166 return gotkids;
167 }
168
169 double get_total_cost() const {
170 return total_cost;
171 }
172
173 /// True if partitioning logically deleted this node, so it is not a subtree root
174 bool is_deleted() const {
175 return deleted;
176 }
177
178 /// Accumulates cost into this node
179 void add(double cost, bool got_kids) {
180 total_cost = (my_cost += cost);
181 gotkids = gotkids || got_kids;
182 }
183
184 /// Accumulates cost up the tree from children
185 void sum(const treeT& tree, const keyT& child, double value) {
186 child_cost[index(child)] = value;
187 ++nsummed;
188 if (nsummed == nchild) {
189 for (int i=0; i<nchild; ++i) total_cost += child_cost[i];
190 if (child.level() > 1) {
191 keyT key = child.parent();
192 keyT parent = key.parent();
193 const_cast<treeT&>(tree).task(parent, &nodeT::sum, tree, key, double(total_cost));
194 }
195 }
196 }
197
198
199 /// Logically deletes this node and its descendants
200
201 /// Cannot actually erase this node from the container since the send() handler
202 /// is holding an accessor to it.
203 void deleter(const treeT& tree, const keyT& key) {
204 deleted = true;
205 if (has_children()) {
206 for (KeyChildIterator<NDIM> kit(key); kit; ++kit) {
207 const keyT child = kit.key();
208 const_cast<treeT&>(tree).task(child, &nodeT::deleter, tree, child);
209 }
210 }
211 }
212
213 /// Descends tree deleting all except internal nodes and sub-tree parents
214 void partition(const treeT& tree, const keyT& key, double avg) {
215 if (has_children()) {
216 // Sort children in descending cost order
217 keyT keys[nchild];
218 double vals[nchild];
219 for (KeyChildIterator<NDIM> kit(key); kit; ++kit) {
220 const keyT child = kit.key();
221 int ind = index(child);
222 keys[ind] = child;
223 vals[ind] = child_cost[ind];
224 }
225 for (int i=0; i<nchild; ++i) {
226 for (int j=i+1; j<nchild; ++j) {
227 if (vals[i] < vals[j]) {
228 std::swap(vals[i],vals[j]);
229 std::swap(keys[i],keys[j]);
230 }
231 }
232 }
233
234 // Split off subtrees in decreasing cost order
235 for (int i=0; i<nchild; ++i) {
236 if (total_cost <= avg) {
237 const_cast<treeT&>(tree).task(keys[i], &nodeT::deleter, tree, keys[i]);
238 }
239 else {
240 total_cost -= vals[i];
241 const_cast<treeT&>(tree).task(keys[i], &nodeT::partition, tree, keys[i], avg);
242 }
243 }
244 }
245 }
246
247 template <typename Archive>
248 void serialize(Archive& ar) {
249 ar & archive::wrap_opaque(this,1);
250 }
251 };
252
253
254 template <std::size_t NDIM>
259 typedef typename treeT::iterator iteratorT;
263
264
265 template <typename T, typename costT>
266 struct add_op {
268 const costT& costfn;
270 void operator()(const keyT& key, const FunctionNode<T,NDIM>& node) const {
271 if (lb->tree.is_local(key))
272 lb->tree.send(key, &nodeT::add, costfn(key,node), node.has_children());
273 else
274 lb->tree.task(key, &nodeT::add, costfn(key,node), node.has_children());
275 }
276 };
277
278 /// Sums costs up the tree returning to everyone the total cost
279 double sum() {
280 world.gop.fence();
281 const_iteratorT end = tree.end();
282 for (const_iteratorT it=tree.begin(); it!=end; ++it) {
283 const keyT& key = it->first;
284 const nodeT& node = it->second;
285 if (!node.has_children() && key.level() > 0) {
286 tree.task(key.parent(), &nodeT::sum, tree, key, node.get_total_cost());
287 }
288 }
289 world.gop.fence();
290 double total;
291 keyT key0(0);
292 if (world.rank() == tree.owner(key0)) {
293 total = tree.find(key0).get()->second.get_total_cost();
294 }
295 world.gop.broadcast(total, tree.owner(key0));
296 world.gop.fence();
297
298 return total;
299 }
300
301 /// Used to sort results into descending order
302 static bool compare(const std::pair<keyT,double>& a, const std::pair<keyT,double>& b) {
303 return a.second < b.second;
304 }
305
306
307 public:
309 : world(world)
310 , tree(world, FunctionDefaults<NDIM>::get_pmap()) {
311 world.gop.fence();
312 };
313
314 /// Accumulates cost from a function
315 template <typename T, typename costT>
316 void add_tree(const Function<T,NDIM>& f, const costT& costfn, bool fence=false) {
317 const_cast<Function<T,NDIM>&>(f).unaryop_node(add_op<T,costT>(this,costfn), fence);
318 }
319
320 /// Printing for the curious
321 void print_tree(const keyT& key = keyT(0)) {
322 Future<iteratorT> futit = tree.find(key);
323 iteratorT it = futit.get();
324 if (it != tree.end()) {
325 for (int i=0; i<key.level(); ++i) std::cout << " ";
326 print(key, it->second.get_total_cost());
327
328 if (it->second.has_children()) {
329 for (KeyChildIterator<NDIM> kit(key); kit; ++kit) {
330 print_tree(kit.key());
331 }
332 }
333 }
334 }
335
336 /// Per-rank cost bar chart under the given pmap.
337 ///
338 /// Walks the aggregator tree locally; each entry that is not deleted
339 /// contributes its total_cost to bucket pmap->owner(key). Buckets are
340 /// summed across ranks and rank 0 prints a horizontal bar chart with
341 /// total/avg/max/avg/max/min ratios.
342 ///
343 /// Call BEFORE load_balance() with the old pmap to see the per-key
344 /// distribution that the partitioner is about to act on; call AFTER
345 /// with the returned pmap to see how the bin-packed subtrees fell out.
346 /// Both invocations sum to the same total work, so the bars are
347 /// directly comparable.
348 void print_cost_per_rank(const std::string& label,
349 const std::shared_ptr< WorldDCPmapInterface<keyT> >& pmap,
350 std::ostream& os = std::cout) {
351 const int nproc = world.size();
352 std::vector<double> cost(nproc, 0.0);
353
354 const_iteratorT end = tree.end();
355 for (const_iteratorT it = tree.begin(); it != end; ++it) {
356 if (it->second.is_deleted()) continue;
357 cost[pmap->owner(it->first)] += it->second.get_total_cost();
358 }
359 world.gop.sum(cost.data(), nproc);
360 world.gop.fence();
361 if (world.rank() != 0) return;
362
363 double total = 0.0;
364 double mx = 0.0;
365 double mn = std::numeric_limits<double>::max();
366 for (double c : cost) {
367 total += c;
368 if (c > mx) mx = c;
369 if (c < mn) mn = c;
370 }
371 const double avg = (nproc > 0) ? total / nproc : 0.0;
372 const int barwidth = 40;
373
374 std::ios::fmtflags saved_flags = os.flags();
375 std::streamsize saved_prec = os.precision();
376
377 os << "=== Load balance summary: " << label << " ===\n";
378 os << std::fixed << std::setprecision(2)
379 << " total=" << total
380 << " avg=" << avg
381 << " max/avg=" << (avg > 0.0 ? mx / avg : 0.0);
382 if (mn > 0.0) os << " max/min=" << mx / mn;
383 os << "\n";
384 for (int p = 0; p < nproc; ++p) {
385 int n = (mx > 0.0) ? static_cast<int>(std::lround(barwidth * cost[p] / mx)) : 0;
386 if (n < 0) n = 0;
387 if (n > barwidth) n = barwidth;
388 os << " rank " << std::setw(3) << p << " |"
389 << std::string(n, '#')
390 << std::string(barwidth - n, ' ')
391 << "| " << std::setw(10) << cost[p]
392 << " (" << (avg > 0.0 ? cost[p] / avg : 0.0) << "x avg)\n";
393 }
394 os << "================================" << std::endl;
395
396 os.flags(saved_flags);
397 os.precision(saved_prec);
398 }
399
400 struct CostPerProc {
401 double cost;
402 int proc;
403 CostPerProc() : cost(0.0), proc(0) {}
404 CostPerProc(double cost, int proc) : cost(cost), proc(proc) {}
405 bool operator<(const CostPerProc& other) const {
406 return cost > other.cost; // Want ascending order
407 }
408 };
409
410 /// Actually does the partitioning of the tree
411 std::shared_ptr< WorldDCPmapInterface<keyT> > load_balance(double fac = 1.0, bool printstuff=false) {
412 world.gop.fence();
413 // Compute full tree of costs
414 const double total = sum();
415 // A NaN or infinite cost anywhere reaches the root total, and would break the sort below.
416 // total is broadcast, so every rank throws together.
417 MADNESS_CHECK_THROW(std::isfinite(total), "LoadBalanceDeux: the cost function returned a non-finite cost");
418 double avg = total/(world.size()*fac);
419 //if (world.rank() == 0) print_tree();
420 world.gop.fence();
421
422 // Create partitioning
423 keyT key0(0);
424 if (world.rank() == tree.owner(key0)) {
425 tree.send(key0, &nodeT::partition, tree, key0, avg*1.1);
426 }
427 world.gop.fence();
428
429 // Collect entire vector onto node0
430 std::vector< std::pair<keyT,double> > results;
431 const_iteratorT end = tree.end();
432 for (const_iteratorT it=tree.begin(); it!=end; ++it) {
433 if (!it->second.is_deleted()) {
434 results.push_back(std::make_pair(it->first,it->second.get_total_cost()));
435 }
436 }
437 results = world.gop.concat0(results, 128*1024*1024);
438 world.gop.fence();
439
440 std::vector< std::pair<keyT,ProcessID> > map;
441
442 if (world.rank() == 0) {
443
444 std::sort(results.begin(), results.end(), compare);
445 if (printstuff) {
446 print("THESE ARE THE INITIAL SUBTREES");
447 for (unsigned int i=0; i<results.size(); ++i) print(i,results[i]);
448 }
449
450 // Now use bin packing to cram the results together
451 map.reserve(results.size());
452
453 // Shove the first nproc entries directly into the queue
454 unsigned int nproc = world.size();
455 std::priority_queue<CostPerProc> costs;
456 for (unsigned int p=0; p<nproc && !results.empty(); ++p) {
457 const std::pair<keyT,double>& f = results.back();
458 costs.push(CostPerProc(f.second,p));
459 map.push_back(std::make_pair(f.first,p));
460 results.pop_back();
461 }
462
463 // Process the remainder using the sorting maintained by the priority queue
464 while (!results.empty()) {
465 const std::pair<keyT,double>& f = results.back();
466 CostPerProc top = costs.top();
467 costs.pop();
468 top.cost += f.second;
469 costs.push(top);
470 map.push_back(std::make_pair(f.first,top.proc));
471 results.pop_back();
472 }
473 if (printstuff) {
474 print("THIS IS THE MAP");
475 print(map);
476 print("THESE ARE THE COSTS PER PROCESSOR");
477 while (!costs.empty()) {
478 print(costs.top().proc,costs.top().cost);
479 costs.pop();
480 }
481 }
482 }
483
484 world.gop.fence();
485 world.gop.broadcast_serializable(map, 0);
486 world.gop.fence();
487
488 // Return the Procmap
489
490 return std::shared_ptr< WorldDCPmapInterface<keyT> >(new LBDeuxPmap<NDIM>(map, world.size()));
491 }
492 };
493}
494
495
496#endif // MADNESS_MRA_IBDEUX_H__INCLUDED
497
Implements AtomicInt.
An integer with atomic set, get, read+increment, read+decrement, and decrement+test operations.
Definition atomicint.h:126
FunctionDefaults holds default paramaters as static class members.
Definition funcdefaults.h:101
FunctionNode holds the coefficients, etc., at each node of the 2^NDIM-tree.
Definition funcimpl.h:136
bool has_children() const
Returns true if this node has children.
Definition funcimpl.h:217
A multiresolution adaptive numerical function.
Definition mra.h:141
A future is a possibly yet unevaluated value.
Definition future.h:370
T & get(bool dowork=true) &
Gets the value, waiting if necessary.
Definition future.h:571
Iterates in lexical order thru all children of a key.
Definition key.h:624
Key is the index for a node of the 2^NDIM-tree.
Definition key.h:71
Level level() const
Definition key.h:170
Key parent(int generation=1) const
Returns the key of the parent.
Definition key.h:366
const Vector< Translation, NDIM > & translation() const
Definition key.h:175
Definition lbdeux.h:65
ProcessID owner(const keyT &key) const
Definition lbdeux.h:94
void print() const
Definition lbdeux.h:108
Level max_level
Deepest level of any entry in map.
Definition lbdeux.h:72
Key< NDIM > keyT
Definition lbdeux.h:66
mapT map
Definition lbdeux.h:71
LBDeuxPmap(const std::vector< pairT > &v, const ProcessID nproc)
Definition lbdeux.h:80
mapT::const_iterator iteratorT
Definition lbdeux.h:73
std::unordered_map< keyT, ProcessID, Hash< keyT > > mapT
Definition lbdeux.h:70
std::pair< keyT, ProcessID > pairT
Definition lbdeux.h:67
Definition lbdeux.h:116
void add(double cost, bool got_kids)
Accumulates cost into this node.
Definition lbdeux.h:179
double get_total_cost() const
Definition lbdeux.h:169
LBNodeDeux()
Definition lbdeux.h:138
void serialize(Archive &ar)
Definition lbdeux.h:248
Key< NDIM > keyT
Definition lbdeux.h:118
void partition(const treeT &tree, const keyT &key, double avg)
Descends tree deleting all except internal nodes and sub-tree parents.
Definition lbdeux.h:214
bool is_deleted() const
True if partitioning logically deleted this node, so it is not a subtree root.
Definition lbdeux.h:174
bool gotkids
Definition lbdeux.h:125
AtomicInt nsummed
Definition lbdeux.h:128
LBNodeDeux< NDIM > & operator=(const LBNodeDeux< NDIM > &other)
Definition lbdeux.h:153
double total_cost
Definition lbdeux.h:124
bool deleted
Set by deleter(). A retained node can have any cost, even a negative one.
Definition lbdeux.h:126
static const int nchild
Definition lbdeux.h:117
bool has_children() const
Definition lbdeux.h:165
WorldContainer< keyT, nodeT > treeT
Definition lbdeux.h:120
int index(const keyT &key)
Computes index of child key in this node using last bit of translations.
Definition lbdeux.h:131
LBNodeDeux(const LBNodeDeux< NDIM > &other)
Definition lbdeux.h:145
double my_cost
Definition lbdeux.h:123
double child_cost[nchild]
Definition lbdeux.h:122
LBNodeDeux< NDIM > nodeT
Definition lbdeux.h:119
void deleter(const treeT &tree, const keyT &key)
Logically deletes this node and its descendants.
Definition lbdeux.h:203
void sum(const treeT &tree, const keyT &child, double value)
Accumulates cost up the tree from children.
Definition lbdeux.h:185
Key< NDIM > keyT
Definition lbdeux.h:256
World & world
Definition lbdeux.h:261
treeT::iterator iteratorT
Definition lbdeux.h:259
treeT tree
Definition lbdeux.h:262
LBNodeDeux< NDIM > nodeT
Definition lbdeux.h:257
std::shared_ptr< WorldDCPmapInterface< keyT > > load_balance(double fac=1.0, bool printstuff=false)
Actually does the partitioning of the tree.
Definition lbdeux.h:411
void print_tree(const keyT &key=keyT(0))
Printing for the curious.
Definition lbdeux.h:321
treeT::const_iterator const_iteratorT
Definition lbdeux.h:260
double sum()
Sums costs up the tree returning to everyone the total cost.
Definition lbdeux.h:279
static bool compare(const std::pair< keyT, double > &a, const std::pair< keyT, double > &b)
Used to sort results into descending order.
Definition lbdeux.h:302
void add_tree(const Function< T, NDIM > &f, const costT &costfn, bool fence=false)
Accumulates cost from a function.
Definition lbdeux.h:316
WorldContainer< keyT, nodeT > treeT
Definition lbdeux.h:258
LoadBalanceDeux(World &world)
Definition lbdeux.h:308
void print_cost_per_rank(const std::string &label, const std::shared_ptr< WorldDCPmapInterface< keyT > > &pmap, std::ostream &os=std::cout)
Definition lbdeux.h:348
Makes a distributed container with specified attributes.
Definition worlddc.h:1302
implT::const_iterator const_iterator
Definition worlddc.h:1310
implT::iterator iterator
Definition worlddc.h:1309
Interface to be provided by any process map.
Definition worlddc.h:128
A parallel world class.
Definition world.h:134
char * p(char *buf, const char *name, int k, int initial_level, double thresh, int order)
Definition derivatives.cc:72
Provides FunctionDefaults and utilities for coordinate transformation.
archive_array< unsigned char > wrap_opaque(const T *, std::size_t)
Factory function to wrap a pointer to contiguous data as an opaque (uchar) archive_array.
Definition archive.h:926
static const double v
Definition hatom_sf_dirac.cc:20
Multidimension Key for MRA tree and associated iterators.
Macros and tools pertaining to the configuration of MADNESS.
#define MADNESS_EXCEPTION(msg, value)
Macro for throwing a MADNESS exception.
Definition madness_exception.h:119
#define MADNESS_CHECK_THROW(condition, msg)
Check a condition — even in a release build the condition is always evaluated so it can have side eff...
Definition madness_exception.h:207
Namespace for all elements and tools of MADNESS.
Definition DFConvergence.h:9
std::shared_ptr< WorldDCPmapInterface< Key< NDIM > > > FunctionDefaults< NDIM >::pmap
Definition mraimpl.h:3898
int FunctionDefaults< NDIM >::k
Definition mraimpl.h:3878
ConcurrentHashMap< hashT, std::shared_ptr< GaussianConvolution1D< double > > > GaussianConvolution1DCache< double >::map
Definition convolution1d.h:1073
int Level
Definition key.h:60
NDIM & f
Definition mra.h:2817
void print(const T &t, Ts &&... ts)
Print items to std::cout (items separated by spaces) and terminate with a new line.
Definition print.h:232
const Future< void > Future< void >::value
Definition future.cc:44
static const double b
Definition nonlinschro.cc:119
static const double d
Definition nonlinschro.cc:121
static const double a
Definition nonlinschro.cc:118
static const double c
Definition relops.cc:10
CostPerProc(double cost, int proc)
Definition lbdeux.h:404
int proc
Definition lbdeux.h:402
bool operator<(const CostPerProc &other) const
Definition lbdeux.h:405
double cost
Definition lbdeux.h:401
CostPerProc()
Definition lbdeux.h:403
Definition lbdeux.h:266
void operator()(const keyT &key, const FunctionNode< T, NDIM > &node) const
Definition lbdeux.h:270
const costT & costfn
Definition lbdeux.h:268
LoadBalanceDeux * lb
Definition lbdeux.h:267
add_op(LoadBalanceDeux *lb, const costT &costfn)
Definition lbdeux.h:269
static constexpr ProcessID nproc
Rank bound for the hand-written maps, which use the ranks 0, 1 and 2.
Definition test_lbdeux.cc:24
int task(int i)
Definition test_runtime.cpp:4
constexpr std::size_t NDIM
Definition testgconv.cc:54
Implements WorldContainer.
int ProcessID
Used to clearly identify process number/rank.
Definition worldtypes.h:43