Communicator performing operations within groups defined by a GroupTopology with arbitrary-size data associated with each group. More...

#include <communication.hpp>

Collaboration diagram for mfem::GroupCommunicator:

[legend]

Classes
struct	OpData
	Data structure on which we define reduce operations. The data is associated with (and the operation is performed on) one group at a time. More...

Public Types
enum	Mode { byGroup , byNeighbor }
	Communication mode. More...

Public Member Functions
	GroupCommunicator (const GroupTopology &gt, Mode m=byNeighbor)
	Construct a GroupCommunicator object.

void	Create (const Array< int > &ldof_group)
	Initialize the communicator from a local-dof to group map. Finalize() is called internally.

Table &	GroupLDofTable ()
	Fill-in the returned Table reference to initialize the GroupCommunicator then call Finalize().

const Table &	GroupLDofTable () const
	Read-only access to group-ldof Table.

void	Finalize ()
	Allocate internal buffers after the GroupLDofTable is defined.

void	SetLTDofTable (const Array< int > &ldof_ltdof)
	Initialize the internal group_ltdof Table.

const GroupTopology &	GetGroupTopology () const
	Get a const reference to the associated GroupTopology object.

void	GetNeighborLTDofTable (Table &nbr_ltdof) const
	Dofs to be sent (during Bcast) to communication neighbors.

void	GetNeighborLDofTable (Table &nbr_ldof) const
	Dofs to be received (during Bcast) from communication neighbors.

const DeviceGroupCommunicator &	GetDeviceComm () const
	Return the device communicator, 'device_gc', constructing it if it was not already constructed.

template<class T >
T *	CopyGroupToBuffer (const T ldata, T buf, int group, int layout) const
	Copy the entries corresponding to the group group from the local array ldata to the buffer buf.

template<class T >
const T *	CopyGroupFromBuffer (const T buf, T ldata, int group, int layout) const
	Copy the entries corresponding to the group group from the buffer buf to the local array ldata.

template<class T >
const T *	ReduceGroupFromBuffer (const T buf, T ldata, int group, int layout, void(*Op)(OpData< T >)) const
	Perform the reduction operation Op to the entries of group group using the values from the buffer buf and the values from the local array ldata, saving the result in the latter.

template<class T >
void	BcastBegin (T *ldata, int layout) const
	Begin a broadcast within each group where the master is the root, host version.

template<class T >
void	BcastBegin (Array< T > &ldata, int layout) const
	Begin a broadcast within each group where the master is the root, device version.

template<class T >
void	BcastEnd (T *ldata, int layout) const
	Finalize a broadcast started with the host version of BcastBegin().

template<class T >
void	BcastEnd (Array< T > &ldata, int layout) const
	Finalize a broadcast started with the device version of BcastBegin().

template<class T >
void	Bcast (T *ldata, int layout) const
	Broadcast within each group where the master is the root.

template<class T >
void	Bcast (T *ldata) const
	Broadcast within each group where the master is the root, host version.

template<class T >
void	Bcast (Array< T > &ldata) const
	Broadcast within each group where the master is the root, device version.

template<class T >
void	ReduceBegin (const T *ldata) const
	Begin reduction operation within each group where the master is the root, host version.

template<class T >
void	ReduceBegin (const Array< T > &ldata, void(*Op)(OpData< T >)=nullptr) const
	Begin reduction operation within each group where the master is the root, device version.

template<class T >
void	ReduceEnd (T ldata, int layout, void(Op)(OpData< T >)) const
	Finalize reduction operation started with the host version of ReduceBegin().

template<class T >
void	ReduceEnd (Array< T > &ldata, int layout, void(*Op)(OpData< T >)) const
	Finalize reduction operation started with the device version of ReduceBegin().

template<class T >
void	Reduce (T ldata, void(Op)(OpData< T >)) const
	Reduce within each group where the master is the root, host version.

template<class T >
void	Reduce (Array< T > &ldata, void(*Op)(OpData< T >)) const
	Reduce within each group where the master is the root, device version.

template<class T >
void	ReduceMarked (T ldata, const Array< int > &marker, int layout, void(Op)(OpData< T >)) const
	Finalize reduction operation started with ReduceBegin(), but only apply the reduction to DOFs marked in the marker array.

template<class T >
void	Reduce (T ldata, const Array< int > &marker, void(Op)(OpData< T >)) const
	Reduce within each group where the master is the root, but only for marked DOFs.

void	PrintInfo (std::ostream &out=mfem::out) const
	Print information about the GroupCommunicator from all MPI ranks.

	~GroupCommunicator ()
	Destroy a GroupCommunicator object, deallocating internal data structures and buffers.

Static Public Member Functions
template<class T >
static void	Sum (OpData< T >)
	Reduce operation Sum, instantiated for int, double and float.

template<class T >
static void	Min (OpData< T >)
	Reduce operation Min, instantiated for int, double and float.

template<class T >
static void	Max (OpData< T >)
	Reduce operation Max, instantiated for int, double and float.

template<class T >
static void	BitOR (OpData< T >)
	Reduce operation bitwise OR, instantiated for int only.

template<class T >
static void	MaxAbs (OpData< T >)

Protected Attributes
const GroupTopology &	gtopo

Mode	mode

Table	group_ldof

Table	group_ltdof

int	group_buf_size

Array< char >	group_buf

MPI_Request *	requests

int	comm_lock

int	num_requests

void(*	reduce_op )()

int *	request_marker

int *	buf_offsets

Table	nbr_send_groups

Table	nbr_recv_groups

bool	have_ltdof_ldof

Array< int >	ltdof_ldof

int	ldof_size

DeviceGroupCommunicator *	device_gc

Friends
class	DeviceGroupCommunicator

Detailed Description

Communicator performing operations within groups defined by a GroupTopology with arbitrary-size data associated with each group.

Definition at line 238 of file communication.hpp.

Member Enumeration Documentation

◆ Mode

enum mfem::GroupCommunicator::Mode

Communication mode.

Enumerator
byGroup	Communications are performed one group at a time.
byNeighbor	Communications are performed one neighbor at a time, aggregating over groups.

Definition at line 244 of file communication.hpp.

Constructor & Destructor Documentation

◆ GroupCommunicator()

DO_NOT_DOCUMENT mfem::GroupCommunicator::GroupCommunicator	(	const GroupTopology &	gt,
		Mode	m = byNeighbor )

Construct a GroupCommunicator object.

The object must be initialized before it can be used to perform any operations. To initialize the object, either

call Create() or
initialize the Table reference returned by GroupLDofTable() and then call Finalize().

Definition at line 371 of file communication.cpp.

◆ ~GroupCommunicator()

mfem::GroupCommunicator::~GroupCommunicator ( )

Destroy a GroupCommunicator object, deallocating internal data structures and buffers.

Definition at line 1746 of file communication.cpp.

Member Function Documentation

◆ Bcast() [1/3]

template<class T >

void mfem::GroupCommunicator::Bcast ( Array< T > & ldata ) const

inline

Broadcast within each group where the master is the root, device version.

The implicit data layout is 0, i.e. the ldata array is an L-dof array.

This method performs the operation on device. However, not all communication modes are supported on device yet. In such cases, the operation is performed on host.

Definition at line 443 of file communication.hpp.

◆ Bcast() [2/3]

template<class T >

void mfem::GroupCommunicator::Bcast ( T * ldata ) const

inline

Broadcast within each group where the master is the root, host version.

The implicit data layout is 0, i.e. the ldata array is an L-dof array.

This method performs the operation on host and expects ldata to be a host pointer.

Definition at line 435 of file communication.hpp.

◆ Bcast() [3/3]

template<class T >

void mfem::GroupCommunicator::Bcast	(	T *	ldata,
		int	layout ) const

inline

Broadcast within each group where the master is the root.

The data layout can be either 0 or 1.

For a description of layout, see CopyGroupToBuffer().

This method performs the operation on host and expects ldata to be a host pointer.

Definition at line 424 of file communication.hpp.

◆ BcastBegin() [1/2]

template<class T >

void mfem::GroupCommunicator::BcastBegin	(	Array< T > &	ldata,
		int	layout ) const

Begin a broadcast within each group where the master is the root, device version.

Parameters

[in,out]	ldata	Input L-vector data; in some cases it is used as a receive buffer, so its type is not const.
[in]	layout	For a description, see CopyGroupToBuffer().

This method performs the operation on device if the device flag of ldata is set. However, not all communication modes and layouts are supported on device yet. In such cases, the operation is performed on host.

Definition at line 889 of file communication.cpp.

◆ BcastBegin() [2/2]

template<class T >

void mfem::GroupCommunicator::BcastBegin	(	T *	ldata,
		int	layout ) const

Begin a broadcast within each group where the master is the root, host version.

Parameters

[in,out]	ldata	Input L-vector data; in some cases it is used as a receive buffer, so its type is not const. It must be a host pointer.
[in]	layout	For a description, see CopyGroupToBuffer().

This method performs the operation on host.

Definition at line 747 of file communication.cpp.

◆ BcastEnd() [1/2]

template<class T >

void mfem::GroupCommunicator::BcastEnd	(	Array< T > &	ldata,
		int	layout ) const

Finalize a broadcast started with the device version of BcastBegin().

Parameters

[out]	ldata	Output L-vector data.
[in]	layout	Output data layout; one of: 0: ldata is an array on all ldofs; the input layout should be either 0 or 2, 1: ldata is the same array as given to BcastBegin(); the input layout should be 1. For a description of the layouts, see CopyGroupToBuffer().

This method performs the operation on device if the device flag of ldata is set. However, not all communication modes and layouts are supported on device yet. In such cases, the operation is performed on host.

It is expected that the device flag of ldata is the same as the device flag of the data array provided to BcastBegin().

Definition at line 1013 of file communication.cpp.

◆ BcastEnd() [2/2]

template<class T >

void mfem::GroupCommunicator::BcastEnd	(	T *	ldata,
		int	layout ) const

Finalize a broadcast started with the host version of BcastBegin().

Parameters

[out]	ldata	Output L-vector data. It must be a host pointer.
[in]	layout	Output data layout; one of: 0: ldata is an array on all ldofs; the input layout should be either 0 or 2, 1: ldata is the same array as given to BcastBegin(); the input layout should be 1. For a description of the layouts, see CopyGroupToBuffer().

This method performs the operation on host.

Definition at line 942 of file communication.cpp.

◆ BitOR()

template<class T >

template void mfem::GroupCommunicator::BitOR< int > ( OpData< T > )

static

Reduce operation bitwise OR, instantiated for int only.

Definition at line 1584 of file communication.cpp.

◆ CopyGroupFromBuffer()

template<class T >

const T * mfem::GroupCommunicator::CopyGroupFromBuffer	(	const T *	buf,
		T *	ldata,
		int	group,
		int	layout ) const

Copy the entries corresponding to the group group from the buffer buf to the local array ldata.

For a description of layout, see CopyGroupToBuffer().

Returns: The pointer buf plus the number of elements in the group.

Definition at line 674 of file communication.cpp.

◆ CopyGroupToBuffer()

template<class T >

T * mfem::GroupCommunicator::CopyGroupToBuffer	(	const T *	ldata,
		T *	buf,
		int	group,
		int	layout ) const

Copy the entries corresponding to the group group from the local array ldata to the buffer buf.

The layout of the local array can be:

0 - ldata is an array on all ldofs: copied indices: { J[j] : I[group] <= j < I[group+1] } where I,J=group_ldof.{I,J}
1 - ldata is an array on the shared ldofs: copied indices: { j : I[group] <= j < I[group+1] } where I,J=group_ldof.{I,J}
2 - ldata is an array on the true ldofs, ltdofs: copied indices: { J[j] : I[group] <= j < I[group+1] } where I,J=group_ltdof.{I,J}.
Returns
The pointer buf plus the number of elements in the group.

Definition at line 639 of file communication.cpp.

◆ Create()

void mfem::GroupCommunicator::Create ( const Array< int > & ldof_group )

Initialize the communicator from a local-dof to group map. Finalize() is called internally.

Definition at line 386 of file communication.cpp.

◆ Finalize()

void mfem::GroupCommunicator::Finalize ( )

Allocate internal buffers after the GroupLDofTable is defined.

Definition at line 415 of file communication.cpp.

◆ GetDeviceComm()

const DeviceGroupCommunicator & mfem::GroupCommunicator::GetDeviceComm ( ) const

Return the device communicator, 'device_gc', constructing it if it was not already constructed.

Definition at line 627 of file communication.cpp.

◆ GetGroupTopology()

const GroupTopology & mfem::GroupCommunicator::GetGroupTopology ( ) const

inline

Get a const reference to the associated GroupTopology object.

Definition at line 302 of file communication.hpp.

◆ GetNeighborLDofTable()

void mfem::GroupCommunicator::GetNeighborLDofTable ( Table & nbr_ldof ) const

Dofs to be received (during Bcast) from communication neighbors.

Definition at line 622 of file communication.cpp.

◆ GetNeighborLTDofTable()

void mfem::GroupCommunicator::GetNeighborLTDofTable ( Table & nbr_ltdof ) const

Dofs to be sent (during Bcast) to communication neighbors.

Definition at line 617 of file communication.cpp.

◆ GroupLDofTable() [1/2]

Table & mfem::GroupCommunicator::GroupLDofTable ( )

inline

Fill-in the returned Table reference to initialize the GroupCommunicator then call Finalize().

Definition at line 288 of file communication.hpp.

◆ GroupLDofTable() [2/2]

const Table & mfem::GroupCommunicator::GroupLDofTable ( ) const

inline

Read-only access to group-ldof Table.

Definition at line 291 of file communication.hpp.

◆ Max()

template<class T >

template void mfem::GroupCommunicator::Max< float > ( OpData< T > )

static

Reduce operation Max, instantiated for int, double and float.

Definition at line 1566 of file communication.cpp.

◆ MaxAbs()

template<class T >

template void mfem::GroupCommunicator::MaxAbs< float > ( OpData< T > )

static

Reduce operation selecting the signed value with the largest absolute value, instantiated for int, double and float. The result keeps its sign; it is not the non-negative absolute value. Equal-magnitude ties are broken deterministically toward the more positive value, so opposite-sign ties resolve to the positive one regardless of accumulation order.

Definition at line 1600 of file communication.cpp.

◆ Min()

template<class T >

template void mfem::GroupCommunicator::Min< float > ( OpData< T > )

static

Reduce operation Min, instantiated for int, double and float.

Definition at line 1548 of file communication.cpp.

◆ PrintInfo()

void mfem::GroupCommunicator::PrintInfo ( std::ostream & out = mfem::out ) const

Print information about the GroupCommunicator from all MPI ranks.

Definition at line 1626 of file communication.cpp.

◆ Reduce() [1/3]

template<class T >

void mfem::GroupCommunicator::Reduce	(	Array< T > &	ldata,
		void(*	Op )(OpData< T >) ) const

inline

Reduce within each group where the master is the root, device version.

The implicit data layout is 0, i.e. the ldata array is an L-dof array.

The reduce operation is given by the second argument (see below for list of the supported operations.)

This method performs the operation on device. However, not all communication modes are supported on device yet. In such cases, the operation is performed on host.

Definition at line 551 of file communication.hpp.

◆ Reduce() [2/3]

template<class T >

void mfem::GroupCommunicator::Reduce	(	T *	ldata,
		const Array< int > &	marker,
		void(*	Op )(OpData< T >) ) const

inline

Reduce within each group where the master is the root, but only for marked DOFs.

Definition at line 585 of file communication.hpp.

◆ Reduce() [3/3]

template<class T >

void mfem::GroupCommunicator::Reduce	(	T *	ldata,
		void(*	Op )(OpData< T >) ) const

inline

Reduce within each group where the master is the root, host version.

The implicit data layout is 0, i.e. the ldata array is an L-dof array.

The reduce operation is given by the second argument (see below for list of the supported operations.)

This method performs the operation on host and expects ldata to be a host pointer.

Definition at line 534 of file communication.hpp.

◆ ReduceBegin() [1/2]

template<class T >

void mfem::GroupCommunicator::ReduceBegin	(	const Array< T > &	ldata,
		void(*	Op )(OpData< T >) = nullptr ) const

Begin reduction operation within each group where the master is the root, device version.

The input data layout is an array on all ldofs, i.e. layout 0, see CopyGroupToBuffer().

Generally, the reduce operation will be specified when calling ReduceEnd(), however, if the reduction operation is not supported on device, it must be given to this call as the optional second argument Op. This method is instantiated for int, double, and float.

This method performs the operation on device if the device flag of ldata is set. However, not all communication modes are supported on device yet. In such cases, the operation is performed on host.

Definition at line 1211 of file communication.cpp.

◆ ReduceBegin() [2/2]

template<class T >

void mfem::GroupCommunicator::ReduceBegin ( const T * ldata ) const

Begin reduction operation within each group where the master is the root, host version.

The input data layout is an array on all ldofs, i.e. layout 0, see CopyGroupToBuffer().

The reduce operation will be specified when calling ReduceEnd(). This method is instantiated for int, double, and float.

This method performs the operation on host and expects ldata to be a host pointer.

Definition at line 1054 of file communication.cpp.

◆ ReduceEnd() [1/2]

template<class T >

void mfem::GroupCommunicator::ReduceEnd	(	Array< T > &	ldata,
		int	layout,
		void(*	Op )(OpData< T >) ) const

Finalize reduction operation started with the device version of ReduceBegin().

The output data layout can be either 0 or 2, see CopyGroupToBuffer().

The reduce operation is given by the third argument (see below for list of the supported operations.) This method is instantiated for int, double, and float.

This method performs the operation on device if the device flag of ldata is set. However, not all communication modes and layouts are supported on device yet. In such cases, the operation is performed on host.

It is expected that the device flag of ldata is the same as the device flag of the data array provided to ReduceBegin().

Note: If the output data layout is 2, then the data from the ldata array passed to this call is used in the reduction operation, instead of the data from the ldata array passed to ReduceBegin(). Therefore, the data for master-groups has to be identical in both arrays.

Definition at line 1337 of file communication.cpp.

◆ ReduceEnd() [2/2]

template<class T >

void mfem::GroupCommunicator::ReduceEnd	(	T *	ldata,
		int	layout,
		void(*	Op )(OpData< T >) ) const

Finalize reduction operation started with the host version of ReduceBegin().

The output data layout can be either 0 or 2, see CopyGroupToBuffer().

The reduce operation is given by the third argument (see below for list of the supported operations.) This method is instantiated for int, double, and float.

This method performs the operation on host and expects ldata to be a host pointer.

Note: If the output data layout is 2, then the data from the ldata array passed to this call is used in the reduction operation, instead of the data from the ldata array passed to ReduceBegin(). Therefore, the data for master-groups has to be identical in both arrays.

Definition at line 1258 of file communication.cpp.

◆ ReduceGroupFromBuffer()

template<class T >

const T * mfem::GroupCommunicator::ReduceGroupFromBuffer	(	const T *	buf,
		T *	ldata,
		int	group,
		int	layout,
		void(*	Op )(OpData< T >) ) const

Perform the reduction operation Op to the entries of group group using the values from the buffer buf and the values from the local array ldata, saving the result in the latter.

For a description of layout, see CopyGroupToBuffer().

Returns: The pointer buf plus the number of elements in the group.

Definition at line 708 of file communication.cpp.

◆ ReduceMarked()

template<class T >

void mfem::GroupCommunicator::ReduceMarked	(	T *	ldata,
		const Array< int > &	marker,
		int	layout,
		void(*	Op )(OpData< T >) ) const

Finalize reduction operation started with ReduceBegin(), but only apply the reduction to DOFs marked in the marker array.

Note: The reduction is carried out in the signed type T, so the result is signed even for bitwise operations.

Definition at line 1404 of file communication.cpp.

◆ SetLTDofTable()

void mfem::GroupCommunicator::SetLTDofTable ( const Array< int > & ldof_ltdof )

Initialize the internal group_ltdof Table.

This method must be called before performing operations that use local data layout 2, see CopyGroupToBuffer() for layout descriptions.

Definition at line 526 of file communication.cpp.

◆ Sum()

template<class T >

template void mfem::GroupCommunicator::Sum< float > ( OpData< T > )

static

Reduce operation Sum, instantiated for int, double and float.

Definition at line 1524 of file communication.cpp.

Friends And Related Symbol Documentation

◆ DeviceGroupCommunicator

friend class DeviceGroupCommunicator

friend

Definition at line 240 of file communication.hpp.

Member Data Documentation

◆ buf_offsets

int* mfem::GroupCommunicator::buf_offsets

protected

Definition at line 265 of file communication.hpp.

◆ comm_lock

int mfem::GroupCommunicator::comm_lock

mutableprotected

Definition at line 261 of file communication.hpp.

◆ device_gc

DeviceGroupCommunicator* mfem::GroupCommunicator::device_gc

mutableprotected

Definition at line 270 of file communication.hpp.

◆ group_buf

Array<char> mfem::GroupCommunicator::group_buf

mutableprotected

Definition at line 257 of file communication.hpp.

◆ group_buf_size

int mfem::GroupCommunicator::group_buf_size

protected

Definition at line 256 of file communication.hpp.

◆ group_ldof

Table mfem::GroupCommunicator::group_ldof

protected

Definition at line 254 of file communication.hpp.

◆ group_ltdof

Table mfem::GroupCommunicator::group_ltdof

protected

Definition at line 255 of file communication.hpp.

◆ gtopo

const GroupTopology& mfem::GroupCommunicator::gtopo

protected

Definition at line 252 of file communication.hpp.

◆ have_ltdof_ldof

bool mfem::GroupCommunicator::have_ltdof_ldof

protected

Definition at line 267 of file communication.hpp.

◆ ldof_size

int mfem::GroupCommunicator::ldof_size

protected

Definition at line 269 of file communication.hpp.

◆ ltdof_ldof

Array<int> mfem::GroupCommunicator::ltdof_ldof

protected

Definition at line 268 of file communication.hpp.

◆ mode

Mode mfem::GroupCommunicator::mode

protected

Definition at line 253 of file communication.hpp.

◆ nbr_recv_groups

Table mfem::GroupCommunicator::nbr_recv_groups

protected

Definition at line 266 of file communication.hpp.

◆ nbr_send_groups

Table mfem::GroupCommunicator::nbr_send_groups

protected

Definition at line 266 of file communication.hpp.

◆ num_requests

int mfem::GroupCommunicator::num_requests

mutableprotected

Definition at line 262 of file communication.hpp.

◆ reduce_op

void(* mfem::GroupCommunicator::reduce_op) ()

mutableprotected

Definition at line 263 of file communication.hpp.

◆ request_marker

int* mfem::GroupCommunicator::request_marker

protected

Definition at line 264 of file communication.hpp.

◆ requests

MPI_Request* mfem::GroupCommunicator::requests

protected

Definition at line 258 of file communication.hpp.

The documentation for this class was generated from the following files:

general/communication.hpp
general/communication.cpp

Classes

Public Types

Public Member Functions

Static Public Member Functions

Protected Attributes

Friends

Detailed Description

Member Enumeration Documentation

◆ Mode

Constructor & Destructor Documentation

◆ GroupCommunicator()

◆ ~GroupCommunicator()

Member Function Documentation

◆ Bcast() [1/3]

◆ Bcast() [2/3]

◆ Bcast() [3/3]

◆ BcastBegin() [1/2]

◆ BcastBegin() [2/2]

◆ BcastEnd() [1/2]

◆ BcastEnd() [2/2]

◆ BitOR()

◆ CopyGroupFromBuffer()

◆ CopyGroupToBuffer()

◆ Create()

◆ Finalize()

◆ GetDeviceComm()

◆ GetGroupTopology()

◆ GetNeighborLDofTable()

◆ GetNeighborLTDofTable()

◆ GroupLDofTable() [1/2]

◆ GroupLDofTable() [2/2]

◆ Max()

◆ MaxAbs()

◆ Min()

◆ PrintInfo()

◆ Reduce() [1/3]

◆ Reduce() [2/3]

◆ Reduce() [3/3]

◆ ReduceBegin() [1/2]

◆ ReduceBegin() [2/2]

◆ ReduceEnd() [1/2]

◆ ReduceEnd() [2/2]

◆ ReduceGroupFromBuffer()

◆ ReduceMarked()

◆ SetLTDofTable()

◆ Sum()

Friends And Related Symbol Documentation

◆ DeviceGroupCommunicator

Member Data Documentation

◆ buf_offsets

◆ comm_lock

◆ device_gc

◆ group_buf

◆ group_buf_size

◆ group_ldof

◆ group_ltdof

◆ gtopo

◆ have_ltdof_ldof

◆ ldof_size

◆ ltdof_ldof

◆ mode

◆ nbr_recv_groups

◆ nbr_send_groups

◆ num_requests

◆ reduce_op

◆ request_marker

◆ requests