MFEM  v3.4
Finite element discretization library
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
communication.hpp
Go to the documentation of this file.
1 // Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
2 // the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
3 // reserved. See file COPYRIGHT for details.
4 //
5 // This file is part of the MFEM library. For more information and source code
6 // availability see http://mfem.org.
7 //
8 // MFEM is free software; you can redistribute it and/or modify it under the
9 // terms of the GNU Lesser General Public License (as published by the Free
10 // Software Foundation) version 2.1 dated February 1999.
11 
12 #ifndef MFEM_COMMUNICATION
13 #define MFEM_COMMUNICATION
14 
15 #include "../config/config.hpp"
16 
17 #ifdef MFEM_USE_MPI
18 
19 #include "array.hpp"
20 #include "table.hpp"
21 #include "sets.hpp"
22 #include "globals.hpp"
23 #include <mpi.h>
24 
25 
26 namespace mfem
27 {
28 
29 /** @brief A simple convenience class that calls MPI_Init() at construction and
30  MPI_Finalize() at destruction. It also provides easy access to
31  MPI_COMM_WORLD's rank and size. */
33 {
34 protected:
36  void GetRankAndSize();
37 public:
38  MPI_Session() { MPI_Init(NULL, NULL); GetRankAndSize(); }
39  MPI_Session(int &argc, char **&argv)
40  { MPI_Init(&argc, &argv); GetRankAndSize(); }
41  ~MPI_Session() { MPI_Finalize(); }
42  /// Return MPI_COMM_WORLD's rank.
43  int WorldRank() const { return world_rank; }
44  /// Return MPI_COMM_WORLD's size.
45  int WorldSize() const { return world_size; }
46  /// Return true if WorldRank() == 0.
47  bool Root() const { return world_rank == 0; }
48 };
49 
51 {
52 private:
53  MPI_Comm MyComm;
54 
55  /* The shared entities (e.g. vertices, faces and edges) are split into
56  groups, each group determined by the set of participating processors.
57  They are numbered locally in lproc. Assumptions:
58  - group 0 is the 'local' group
59  - groupmaster_lproc[0] = 0
60  - lproc_proc[0] = MyRank */
61 
62  // Neighbor ids (lproc) in each group.
63  Table group_lproc;
64  // Master neighbor id for each group.
65  Array<int> groupmaster_lproc;
66  // MPI rank of each neighbor.
67  Array<int> lproc_proc;
68  // Group --> Group number in the master.
69  Array<int> group_mgroup;
70 
71  void ProcToLProc();
72 
73 public:
74  GroupTopology() : MyComm(0) {}
75  GroupTopology(MPI_Comm comm) { MyComm = comm; }
76 
77  /// Copy constructor
78  GroupTopology(const GroupTopology &gt);
79  void SetComm(MPI_Comm comm) { MyComm = comm; }
80 
81  MPI_Comm GetComm() const { return MyComm; }
82  int MyRank() const { int r; MPI_Comm_rank(MyComm, &r); return r; }
83  int NRanks() const { int s; MPI_Comm_size(MyComm, &s); return s; }
84 
85  void Create(ListOfIntegerSets &groups, int mpitag);
86 
87  int NGroups() const { return group_lproc.Size(); }
88  // return the number of neighbors including the local processor
89  int GetNumNeighbors() const { return lproc_proc.Size(); }
90  int GetNeighborRank(int i) const { return lproc_proc[i]; }
91  // am I master for group 'g'?
92  bool IAmMaster(int g) const { return (groupmaster_lproc[g] == 0); }
93  // return the neighbor index of the group master for a given group.
94  // neighbor 0 is the local processor
95  int GetGroupMaster(int g) const { return groupmaster_lproc[g]; }
96  // return the rank of the group master for a given group
97  int GetGroupMasterRank(int g) const
98  { return lproc_proc[groupmaster_lproc[g]]; }
99  // for a given group return the group number in the master
100  int GetGroupMasterGroup(int g) const { return group_mgroup[g]; }
101  // get the number of processors in a group
102  int GetGroupSize(int g) const { return group_lproc.RowSize(g); }
103  // return a pointer to a list of neighbors for a given group.
104  // neighbor 0 is the local processor
105  const int *GetGroup(int g) const { return group_lproc.GetRow(g); }
106 
107  /// Save the data in a stream.
108  void Save(std::ostream &out) const;
109  /// Load the data from a stream.
110  void Load(std::istream &in);
111 
112  /// Copy
113  void Copy(GroupTopology & copy) const;
114 
115  virtual ~GroupTopology() {}
116 };
117 
118 /** @brief Communicator performing operations within groups defined by a
119  GroupTopology with arbitrary-size data associated with each group. */
121 {
122 public:
123  /// Communication mode.
124  enum Mode
125  {
126  byGroup, ///< Communications are performed one group at a time.
127  byNeighbor /**< Communications are performed one neighbor at a time,
128  aggregating over groups. */
129  };
130 
131 protected:
135  Table group_ltdof; // only for groups for which this processor is master.
138  MPI_Request *requests;
139  // MPI_Status *statuses;
140  // comm_lock: 0 - no lock, 1 - locked for Bcast, 2 - locked for Reduce
141  mutable int comm_lock;
142  mutable int num_requests;
144  int *buf_offsets; // size = max(number of groups, number of neighbors)
146 
147 public:
148  /// Construct a GroupCommunicator object.
149  /** The object must be initialized before it can be used to perform any
150  operations. To initialize the object, either
151  - call Create() or
152  - initialize the Table reference returned by GroupLDofTable() and then
153  call Finalize().
154  */
156 
157  /** @brief Initialize the communicator from a local-dof to group map.
158  Finalize() is called internally. */
159  void Create(const Array<int> &ldof_group);
160 
161  /** @brief Fill-in the returned Table reference to initialize the
162  GroupCommunicator then call Finalize(). */
164 
165  /// Read-only access to group-ldof Table.
166  const Table &GroupLDofTable() const { return group_ldof; }
167 
168  /// Allocate internal buffers after the GroupLDofTable is defined
169  void Finalize();
170 
171  /// Initialize the internal group_ltdof Table.
172  /** This method must be called before performing operations that use local
173  data layout 2, see CopyGroupToBuffer() for layout descriptions. */
174  void SetLTDofTable(const Array<int> &ldof_ltdof);
175 
176  /// Get a reference to the associated GroupTopology object
178 
179  /// Get a const reference to the associated GroupTopology object
180  const GroupTopology &GetGroupTopology() const { return gtopo; }
181 
182  /** @brief Data structure on which we define reduce operations.
183 
184  The data is associated with (and the operation is performed on) one group
185  at a time. */
186  template <class T> struct OpData
187  {
188  int nldofs, nb;
189  const int *ldofs;
190  T *ldata, *buf;
191  };
192 
193  /** @brief Copy the entries corresponding to the group @a group from the
194  local array @a ldata to the buffer @a buf. */
195  /** The @a layout of the local array can be:
196  - 0 - @a ldata is an array on all ldofs: copied indices:
197  `{ J[j] : I[group] <= j < I[group+1] }` where `I,J=group_ldof.{I,J}`
198  - 1 - @a ldata is an array on the shared ldofs: copied indices:
199  `{ j : I[group] <= j < I[group+1] }` where `I,J=group_ldof.{I,J}`
200  - 2 - @a ldata is an array on the true ldofs, ltdofs: copied indices:
201  `{ J[j] : I[group] <= j < I[group+1] }` where `I,J=group_ltdof.{I,J}`.
202  @returns The pointer @a buf plus the number of elements in the group. */
203  template <class T>
204  T *CopyGroupToBuffer(const T *ldata, T *buf, int group, int layout) const;
205 
206  /** @brief Copy the entries corresponding to the group @a group from the
207  buffer @a buf to the local array @a ldata. */
208  /** For a description of @a layout, see CopyGroupToBuffer().
209  @returns The pointer @a buf plus the number of elements in the group. */
210  template <class T>
211  const T *CopyGroupFromBuffer(const T *buf, T *ldata, int group,
212  int layout) const;
213 
214  /** @brief Perform the reduction operation @a Op to the entries of group
215  @a group using the values from the buffer @a buf and the values from the
216  local array @a ldata, saving the result in the latter. */
217  /** For a description of @a layout, see CopyGroupToBuffer().
218  @returns The pointer @a buf plus the number of elements in the group. */
219  template <class T>
220  const T *ReduceGroupFromBuffer(const T *buf, T *ldata, int group,
221  int layout, void (*Op)(OpData<T>)) const;
222 
223  /// Begin a broadcast within each group where the master is the root.
224  /** For a description of @a layout, see CopyGroupToBuffer(). */
225  template <class T> void BcastBegin(T *ldata, int layout) const;
226 
227  /** @brief Finalize a broadcast started with BcastBegin().
228 
229  The output data @a layout can be:
230  - 0 - @a ldata is an array on all ldofs; the input layout should be
231  either 0 or 2
232  - 1 - @a ldata is the same array as given to BcastBegin(); the input
233  layout should be 1.
234 
235  For more details about @a layout, see CopyGroupToBuffer(). */
236  template <class T> void BcastEnd(T *ldata, int layout) const;
237 
238  /** @brief Broadcast within each group where the master is the root.
239 
240  The data @a layout can be either 0 or 1.
241 
242  For a description of @a layout, see CopyGroupToBuffer(). */
243  template <class T> void Bcast(T *ldata, int layout) const
244  {
245  BcastBegin(ldata, layout);
246  BcastEnd(ldata, layout);
247  }
248 
249  /// Broadcast within each group where the master is the root.
250  template <class T> void Bcast(T *ldata) const { Bcast<T>(ldata, 0); }
251  /// Broadcast within each group where the master is the root.
252  template <class T> void Bcast(Array<T> &ldata) const
253  { Bcast<T>((T *)ldata); }
254 
255  /** @brief Begin reduction operation within each group where the master is
256  the root. */
257  /** The input data layout is an array on all ldofs, i.e. layout 0, see
258  CopyGroupToBuffer().
259 
260  The reduce operation will be specified when calling ReduceEnd(). This
261  method is instantiated for int and double. */
262  template <class T> void ReduceBegin(const T *ldata) const;
263 
264  /** @brief Finalize reduction operation started with ReduceBegin().
265 
266  The output data @a layout can be either 0 or 2, see CopyGroupToBuffer().
267 
268  The reduce operation is given by the third argument (see below for list
269  of the supported operations.) This method is instantiated for int and
270  double.
271 
272  @note If the output data layout is 2, then the data from the @a ldata
273  array passed to this call is used in the reduction operation, instead of
274  the data from the @a ldata array passed to ReduceBegin(). Therefore, the
275  data for master-groups has to be identical in both arrays.
276  */
277  template <class T> void ReduceEnd(T *ldata, int layout,
278  void (*Op)(OpData<T>)) const;
279 
280  /** @brief Reduce within each group where the master is the root.
281 
282  The reduce operation is given by the second argument (see below for list
283  of the supported operations.) */
284  template <class T> void Reduce(T *ldata, void (*Op)(OpData<T>)) const
285  {
286  ReduceBegin(ldata);
287  ReduceEnd(ldata, 0, Op);
288  }
289 
290  /// Reduce within each group where the master is the root.
291  template <class T> void Reduce(Array<T> &ldata, void (*Op)(OpData<T>)) const
292  { Reduce<T>((T *)ldata, Op); }
293 
294  /// Reduce operation Sum, instantiated for int and double
295  template <class T> static void Sum(OpData<T>);
296  /// Reduce operation Min, instantiated for int and double
297  template <class T> static void Min(OpData<T>);
298  /// Reduce operation Max, instantiated for int and double
299  template <class T> static void Max(OpData<T>);
300  /// Reduce operation bitwise OR, instantiated for int only
301  template <class T> static void BitOR(OpData<T>);
302 
303  /// Print information about the GroupCommunicator from all MPI ranks.
304  void PrintInfo(std::ostream &out = mfem::out) const;
305 
306  /** @brief Destroy a GroupCommunicator object, deallocating internal data
307  structures and buffers. */
309 };
310 
311 
312 /// \brief Variable-length MPI message containing unspecific binary data.
313 template<int Tag>
315 {
316  std::string data;
317  MPI_Request send_request;
318 
319  /// Non-blocking send to processor 'rank'.
320  void Isend(int rank, MPI_Comm comm)
321  {
322  Encode(rank);
323  MPI_Isend((void*) data.data(), data.length(), MPI_BYTE, rank, Tag, comm,
324  &send_request);
325  }
326 
327  /// Helper to send all messages in a rank-to-message map container.
328  template<typename MapT>
329  static void IsendAll(MapT& rank_msg, MPI_Comm comm)
330  {
331  typename MapT::iterator it;
332  for (it = rank_msg.begin(); it != rank_msg.end(); ++it)
333  {
334  it->second.Isend(it->first, comm);
335  }
336  }
337 
338  /// Helper to wait for all messages in a map container to be sent.
339  template<typename MapT>
340  static void WaitAllSent(MapT& rank_msg)
341  {
342  typename MapT::iterator it;
343  for (it = rank_msg.begin(); it != rank_msg.end(); ++it)
344  {
345  MPI_Wait(&it->second.send_request, MPI_STATUS_IGNORE);
346  it->second.Clear();
347  }
348  }
349 
350  /** Blocking probe for incoming message of this type from any rank.
351  Returns the rank and message size. */
352  static void Probe(int &rank, int &size, MPI_Comm comm)
353  {
354  MPI_Status status;
355  MPI_Probe(MPI_ANY_SOURCE, Tag, comm, &status);
356  rank = status.MPI_SOURCE;
357  MPI_Get_count(&status, MPI_BYTE, &size);
358  }
359 
360  /** Non-blocking probe for incoming message of this type from any rank.
361  If there is an incoming message, returns true and sets 'rank' and 'size'.
362  Otherwise returns false. */
363  static bool IProbe(int &rank, int &size, MPI_Comm comm)
364  {
365  int flag;
366  MPI_Status status;
367  MPI_Iprobe(MPI_ANY_SOURCE, Tag, comm, &flag, &status);
368  if (!flag) { return false; }
369 
370  rank = status.MPI_SOURCE;
371  MPI_Get_count(&status, MPI_BYTE, &size);
372  return true;
373  }
374 
375  /// Post-probe receive from processor 'rank' of message size 'size'.
376  void Recv(int rank, int size, MPI_Comm comm)
377  {
378  MFEM_ASSERT(size >= 0, "");
379  data.resize(size);
380  MPI_Status status;
381  MPI_Recv((void*) data.data(), size, MPI_BYTE, rank, Tag, comm, &status);
382 #ifdef MFEM_DEBUG
383  int count;
384  MPI_Get_count(&status, MPI_BYTE, &count);
385  MFEM_VERIFY(count == size, "");
386 #endif
387  Decode(rank);
388  }
389 
390  /// Like Recv(), but throw away the messsage.
391  void RecvDrop(int rank, int size, MPI_Comm comm)
392  {
393  data.resize(size);
394  MPI_Status status;
395  MPI_Recv((void*) data.data(), size, MPI_BYTE, rank, Tag, comm, &status);
396  data.resize(0); // don't decode
397  }
398 
399  /// Helper to receive all messages in a rank-to-message map container.
400  template<typename MapT>
401  static void RecvAll(MapT& rank_msg, MPI_Comm comm)
402  {
403  int recv_left = rank_msg.size();
404  while (recv_left > 0)
405  {
406  int rank, size;
407  Probe(rank, size, comm);
408  MFEM_ASSERT(rank_msg.find(rank) != rank_msg.end(), "Unexpected message"
409  " (tag " << Tag << ") from rank " << rank);
410  // NOTE: no guard against receiving two messages from the same rank
411  rank_msg[rank].Recv(rank, size, comm);
412  --recv_left;
413  }
414  }
415 
416  VarMessage() : send_request(MPI_REQUEST_NULL) {}
417  void Clear() { data.clear(); send_request = MPI_REQUEST_NULL; }
418 
419  virtual ~VarMessage()
420  {
421  MFEM_ASSERT(send_request == MPI_REQUEST_NULL,
422  "WaitAllSent was not called after Isend");
423  }
424 
425  VarMessage(const VarMessage &other)
426  : data(other.data), send_request(other.send_request)
427  {
428  MFEM_ASSERT(send_request == MPI_REQUEST_NULL,
429  "Cannot copy message with a pending send.");
430  }
431 
432 protected:
433  virtual void Encode(int rank) {}
434  virtual void Decode(int rank) {}
435 };
436 
437 
438 /// Helper struct to convert a C++ type to an MPI type
439 template <typename Type> struct MPITypeMap;
440 
441 // Specializations of MPITypeMap; mpi_type initialized in communication.cpp:
442 template<> struct MPITypeMap<int> { static const MPI_Datatype mpi_type; };
443 template<> struct MPITypeMap<double> { static const MPI_Datatype mpi_type; };
444 
445 
446 /** Reorder MPI ranks to follow the Z-curve within the physical machine topology
447  (provided that functions to query physical node coordinates are available).
448  Returns a new communicator with reordered ranks. */
449 MPI_Comm ReorderRanksZCurve(MPI_Comm comm);
450 
451 
452 } // namespace mfem
453 
454 #endif
455 
456 #endif
int WorldSize() const
Return MPI_COMM_WORLD&#39;s size.
int GetGroupMasterRank(int g) const
void Create(ListOfIntegerSets &groups, int mpitag)
void ReduceEnd(T *ldata, int layout, void(*Op)(OpData< T >)) const
Finalize reduction operation started with ReduceBegin().
int Size() const
Logical size of the array.
Definition: array.hpp:133
void Reduce(T *ldata, void(*Op)(OpData< T >)) const
Reduce within each group where the master is the root.
int GetGroupMasterGroup(int g) const
void Bcast(T *ldata, int layout) const
Broadcast within each group where the master is the root.
Helper struct to convert a C++ type to an MPI type.
MPI_Comm ReorderRanksZCurve(MPI_Comm comm)
GroupTopology(MPI_Comm comm)
void Create(const Array< int > &ldof_group)
Initialize the communicator from a local-dof to group map. Finalize() is called internally.
int GetGroupSize(int g) const
static const MPI_Datatype mpi_type
const int * GetGroup(int g) const
const Table & GroupLDofTable() const
Read-only access to group-ldof Table.
void GetRow(int i, Array< int > &row) const
Return row i in array row (the Table must be finalized)
Definition: table.cpp:189
void RecvDrop(int rank, int size, MPI_Comm comm)
Like Recv(), but throw away the messsage.
const T * CopyGroupFromBuffer(const T *buf, T *ldata, int group, int layout) const
Copy the entries corresponding to the group group from the buffer buf to the local array ldata...
void Recv(int rank, int size, MPI_Comm comm)
Post-probe receive from processor &#39;rank&#39; of message size &#39;size&#39;.
void BcastEnd(T *ldata, int layout) const
Finalize a broadcast started with BcastBegin().
bool IAmMaster(int g) const
VarMessage(const VarMessage &other)
void Bcast(T *ldata) const
Broadcast within each group where the master is the root.
A simple convenience class that calls MPI_Init() at construction and MPI_Finalize() at destruction...
Communicator performing operations within groups defined by a GroupTopology with arbitrary-size data ...
static void Sum(OpData< T >)
Reduce operation Sum, instantiated for int and double.
int GetNumNeighbors() const
void Finalize()
Allocate internal buffers after the GroupLDofTable is defined.
MPI_Comm GetComm() const
bool Root() const
Return true if WorldRank() == 0.
T * CopyGroupToBuffer(const T *ldata, T *buf, int group, int layout) const
Copy the entries corresponding to the group group from the local array ldata to the buffer buf...
MPI_Request send_request
virtual void Encode(int rank)
static void Min(OpData< T >)
Reduce operation Min, instantiated for int and double.
static void IsendAll(MapT &rank_msg, MPI_Comm comm)
Helper to send all messages in a rank-to-message map container.
void Reduce(Array< T > &ldata, void(*Op)(OpData< T >)) const
Reduce within each group where the master is the root.
int Size() const
Returns the number of TYPE I elements.
Definition: table.hpp:92
GroupCommunicator(GroupTopology &gt, Mode m=byNeighbor)
Construct a GroupCommunicator object.
GroupTopology & GetGroupTopology()
Get a reference to the associated GroupTopology object.
int GetGroupMaster(int g) const
void Copy(GroupTopology &copy) const
Copy.
void Save(std::ostream &out) const
Save the data in a stream.
int WorldRank() const
Return MPI_COMM_WORLD&#39;s rank.
const T * ReduceGroupFromBuffer(const T *buf, T *ldata, int group, int layout, void(*Op)(OpData< T >)) const
Perform the reduction operation Op to the entries of group group using the values from the buffer buf...
virtual void Decode(int rank)
int GetNeighborRank(int i) const
static const MPI_Datatype mpi_type
static void Max(OpData< T >)
Reduce operation Max, instantiated for int and double.
Communications are performed one group at a time.
static void RecvAll(MapT &rank_msg, MPI_Comm comm)
Helper to receive all messages in a rank-to-message map container.
void SetLTDofTable(const Array< int > &ldof_ltdof)
Initialize the internal group_ltdof Table.
Table & GroupLDofTable()
Fill-in the returned Table reference to initialize the GroupCommunicator then call Finalize()...
void BcastBegin(T *ldata, int layout) const
Begin a broadcast within each group where the master is the root.
Data structure on which we define reduce operations.
~GroupCommunicator()
Destroy a GroupCommunicator object, deallocating internal data structures and buffers.
Mode
Communication mode.
MPI_Session(int &argc, char **&argv)
void Isend(int rank, MPI_Comm comm)
Non-blocking send to processor &#39;rank&#39;.
void Bcast(Array< T > &ldata) const
Broadcast within each group where the master is the root.
int NGroups() const
void ReduceBegin(const T *ldata) const
Begin reduction operation within each group where the master is the root.
void SetComm(MPI_Comm comm)
static void WaitAllSent(MapT &rank_msg)
Helper to wait for all messages in a map container to be sent.
int RowSize(int i) const
Definition: table.hpp:108
static bool IProbe(int &rank, int &size, MPI_Comm comm)
List of integer sets.
Definition: sets.hpp:54
OutStream out(std::cout)
Global stream used by the library for standard output. Initially it uses the same std::streambuf as s...
Definition: globals.hpp:64
void PrintInfo(std::ostream &out=mfem::out) const
Print information about the GroupCommunicator from all MPI ranks.
Variable-length MPI message containing unspecific binary data.
void Load(std::istream &in)
Load the data from a stream.
const GroupTopology & GetGroupTopology() const
Get a const reference to the associated GroupTopology object.
static void Probe(int &rank, int &size, MPI_Comm comm)
static void BitOR(OpData< T >)
Reduce operation bitwise OR, instantiated for int only.