MFEM  v3.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
communication.hpp
Go to the documentation of this file.
1 // Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
2 // the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
3 // reserved. See file COPYRIGHT for details.
4 //
5 // This file is part of the MFEM library. For more information and source code
6 // availability see http://mfem.googlecode.com.
7 //
8 // MFEM is free software; you can redistribute it and/or modify it under the
9 // terms of the GNU Lesser General Public License (as published by the Free
10 // Software Foundation) version 2.1 dated February 1999.
11 
12 #ifndef MFEM_COMMUNICATION
13 #define MFEM_COMMUNICATION
14 
15 #include "../config/config.hpp"
16 
17 #ifdef MFEM_USE_MPI
18 
19 #include <mpi.h>
20 #include "array.hpp"
21 #include "table.hpp"
22 #include "sets.hpp"
23 
24 namespace mfem
25 {
26 
28 {
29 private:
30  MPI_Comm MyComm;
31 
32  /* The shared entities (e.g. vertices, faces and edges) are split into
33  groups, each group determined by the set of participating processors.
34  They are numbered locally in lproc. Assumptions:
35  - group 0 is the 'local' group
36  - groupmaster_lproc[0] = 0
37  - lproc_proc[0] = MyRank */
38  Table group_lproc;
39  Array<int> groupmaster_lproc;
40  Array<int> lproc_proc;
41  Array<int> group_mgroup; // group --> group number in the master
42 
43  void ProcToLProc();
44 
45 public:
46  GroupTopology(MPI_Comm comm) { MyComm = comm; }
47 
48  MPI_Comm GetComm() { return MyComm; }
49  int MyRank() { int r; MPI_Comm_rank(MyComm, &r); return r; }
50  int NRanks() { int s; MPI_Comm_size(MyComm, &s); return s; }
51 
52  void Create(ListOfIntegerSets &groups, int mpitag);
53 
54  int NGroups() const { return group_lproc.Size(); }
55  // return the number of neighbors including the local processor
56  int GetNumNeighbors() const { return lproc_proc.Size(); }
57  int GetNeighborRank(int i) const { return lproc_proc[i]; }
58  // am I master for group 'g'?
59  bool IAmMaster(int g) const { return (groupmaster_lproc[g] == 0); }
60  // return the neighbor index of the group master for a given group.
61  // neighbor 0 is the local processor
62  int GetGroupMaster(int g) const { return groupmaster_lproc[g]; }
63  // return the rank of the group master for a given group
64  int GetGroupMasterRank(int g) const
65  { return lproc_proc[groupmaster_lproc[g]]; }
66  // for a given group return the group number in the master
67  int GetGroupMasterGroup(int g) const { return group_mgroup[g]; }
68  // get the number of processors in a group
69  int GetGroupSize(int g) const { return group_lproc.RowSize(g); }
70  // return a pointer to a list of neighbors for a given group.
71  // neighbor 0 is the local processor
72  const int *GetGroup(int g) const { return group_lproc.GetRow(g); }
73 };
74 
76 {
77 private:
78  GroupTopology &gtopo;
79  Table group_ldof;
80  int group_buf_size;
81  Array<char> group_buf;
82  MPI_Request *requests;
83  MPI_Status *statuses;
84 
87  template <class T> static inline MPI_Datatype Get_MPI_Datatype();
88 
89 public:
93  void Create(Array<int> &ldof_group);
96  Table &GroupLDofTable() { return group_ldof; }
98  void Finalize();
99 
101  GroupTopology & GetGroupTopology() { return gtopo; }
102 
105  template <class T> void Bcast(T *ldata);
106  template <class T> void Bcast(Array<T> &ldata) { Bcast<T>((T *)ldata); }
107 
111  template <class T> struct OpData
112  {
113  int nldofs, nb, *ldofs;
114  T *ldata, *buf;
115  };
116 
120  template <class T> void Reduce(T *ldata, void (*Op)(OpData<T>));
121  template <class T> void Reduce(Array<T> &ldata, void (*Op)(OpData<T>))
122  { Reduce<T>((T *)ldata, Op); }
123 
125  template <class T> static void Sum(OpData<T>);
127  template <class T> static void Min(OpData<T>);
129  template <class T> static void Max(OpData<T>);
131  template <class T> static void BitOR(OpData<T>);
132 
134 };
135 
136 }
137 
138 #endif
139 
140 #endif
int GetGroupMasterRank(int g) const
void Create(ListOfIntegerSets &groups, int mpitag)
int Size() const
Logical size of the array.
Definition: array.hpp:108
void Bcast(Array< T > &ldata)
int GetGroupMasterGroup(int g) const
GroupCommunicator(GroupTopology &gt)
GroupTopology(MPI_Comm comm)
int GetGroupSize(int g) const
const int * GetGroup(int g) const
void GetRow(int i, Array< int > &row) const
Return row i in array row (the Table must be finalized)
Definition: table.cpp:148
void Reduce(Array< T > &ldata, void(*Op)(OpData< T >))
bool IAmMaster(int g) const
static void Sum(OpData< T >)
Reduce operation Sum, instantiated for int and double.
int GetNumNeighbors() const
void Finalize()
Allocate internal buffers after the GroupLDofTable is defined.
static void Min(OpData< T >)
Reduce operation Min, instantiated for int and double.
int Size() const
Returns the number of TYPE I elements.
Definition: table.hpp:66
GroupTopology & GetGroupTopology()
Get a reference to the group topology object.
int GetGroupMaster(int g) const
int GetNeighborRank(int i) const
static void Max(OpData< T >)
Reduce operation Max, instantiated for int and double.
void Reduce(T *ldata, void(*Op)(OpData< T >))
int NGroups() const
void Create(Array< int > &ldof_group)
int RowSize(int i) const
Definition: table.hpp:82
List of integer sets.
Definition: sets.hpp:49
static void BitOR(OpData< T >)
Reduce operation bitwise OR, instantiated for int only.