MFEM  v4.2.0
Finite element discretization library
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
mem_manager.hpp
Go to the documentation of this file.
1 // Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
2 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
3 // LICENSE and NOTICE for details. LLNL-CODE-806117.
4 //
5 // This file is part of the MFEM library. For more information and source code
6 // availability visit https://mfem.org.
7 //
8 // MFEM is free software; you can redistribute it and/or modify it under the
9 // terms of the BSD-3 license. We welcome feedback and contributions, see file
10 // CONTRIBUTING.md for details.
11 
12 #ifndef MFEM_MEM_MANAGER_HPP
13 #define MFEM_MEM_MANAGER_HPP
14 
15 #include "globals.hpp"
16 #include "error.hpp"
17 #include <cstring> // std::memcpy
18 #include <type_traits> // std::is_const
19 #include <cstddef> // std::max_align_t
20 
21 namespace mfem
22 {
23 
24 // Implementation of MFEM's lightweight device/host memory manager designed to
25 // work seamlessly with the OCCA, RAJA, and other kernels supported by MFEM.
26 
27 /// Memory types supported by MFEM.
28 enum class MemoryType
29 {
30  HOST, ///< Host memory; using new[] and delete[]
31  HOST_32, ///< Host memory; aligned at 32 bytes
32  HOST_64, ///< Host memory; aligned at 64 bytes
33  HOST_DEBUG, ///< Host memory; allocated from a "host-debug" pool
34  HOST_UMPIRE, ///< Host memory; using Umpire
35  MANAGED, /**< Managed memory; using CUDA or HIP *MallocManaged
36  and *Free */
37  DEVICE, ///< Device memory; using CUDA or HIP *Malloc and *Free
38  DEVICE_DEBUG, /**< Pseudo-device memory; allocated on host from a
39  "device-debug" pool */
40  DEVICE_UMPIRE, ///< Device memory; using Umpire
41  SIZE ///< Number of host and device memory types
42 };
43 
44 /// Static casts to 'int' and sizes of some useful memory types.
45 constexpr int MemoryTypeSize = static_cast<int>(MemoryType::SIZE);
46 constexpr int HostMemoryType = static_cast<int>(MemoryType::HOST);
47 constexpr int HostMemoryTypeSize = static_cast<int>(MemoryType::DEVICE);
48 constexpr int DeviceMemoryType = static_cast<int>(MemoryType::MANAGED);
50 
51 /// Memory type names, used during Device:: configuration.
52 extern const char *MemoryTypeName[MemoryTypeSize];
53 
54 /// Memory classes identify sets of memory types.
55 /** This type is used by kernels that can work with multiple MemoryType%s.
56  * For example, kernels that can use DEVICE or MANAGED memory types should
57  * use MemoryClass::DEVICE for their inputs. */
58 enum class MemoryClass
59 {
60  HOST, /**< Memory types: { HOST, HOST_32, HOST_64, HOST_DEBUG,
61  HOST_UMPIRE, MANAGED } */
62  HOST_32, ///< Memory types: { HOST_32, HOST_64, HOST_DEBUG }
63  HOST_64, ///< Memory types: { HOST_64, HOST_DEBUG }
64  DEVICE, ///< Memory types: { DEVICE, DEVICE_DEBUG, DEVICE_UMPIRE, MANAGED }
65  MANAGED ///< Memory types: { MANAGED }
66 };
67 
68 /// Return true if the given memory type is in MemoryClass::HOST.
69 inline bool IsHostMemory(MemoryType mt) { return mt <= MemoryType::MANAGED; }
70 inline bool IsDeviceMemory(MemoryType mt) { return mt >= MemoryType::MANAGED; }
71 
72 /// Return a suitable MemoryType for a given MemoryClass.
74 
75 /// Return a suitable MemoryClass from a pair of MemoryClass%es.
76 /** Note: this operation is commutative, i.e. a*b = b*a, associative, i.e.
77  (a*b)*c = a*(b*c), and has an identity element: MemoryClass::HOST.
78 
79  Currently, the operation is defined as a*b := max(a,b) where the max
80  operation is based on the enumeration ordering:
81 
82  HOST < HOST_32 < HOST_64 < DEVICE < MANAGED. */
84 
85 /// Class used by MFEM to store pointers to host and/or device memory.
86 /** The template class parameter, T, must be a plain-old-data (POD) type.
87 
88  In many respects this class behaves like a pointer:
89  * When destroyed, a Memory object does NOT automatically delete any
90  allocated memory.
91  * Only the method Delete() will deallocate a Memory object.
92  * Other methods that modify the object (e.g. New(), Wrap(), etc) will simply
93  overwrite the old contents.
94  * One difference with a pointer is that a const Memory object does not allow
95  modification of the content (unlike e.g. a const pointer).
96 
97  A Memory object stores up to two different pointers: one host pointer (with
98  MemoryType from MemoryClass::HOST) and one device pointer (currently one of
99  MemoryType: DEVICE, DEVICE_DEBUG, DEVICE_UMPIRE or MANAGED).
100 
101  A Memory object can hold (wrap) an externally allocated pointer with any
102  given MemoryType.
103 
104  Access to the content of the Memory object can be requested with any given
105  MemoryClass through the methods ReadWrite(), Read(), and Write().
106  Requesting such access may result in additional (internally handled)
107  memory allocation and/or memory copy.
108  * When ReadWrite() is called, the returned pointer becomes the only
109  valid pointer.
110  * When Read() is called, the returned pointer becomes valid, however
111  the other pointer (host or device) may remain valid as well.
112  * When Write() is called, the returned pointer becomes the only valid
113  pointer, however, unlike ReadWrite(), no memory copy will be performed.
114 
115  The host memory (pointer from MemoryClass::HOST) can be accessed through the
116  inline methods: `operator[]()`, `operator*()`, the implicit conversion
117  functions `operator T*()`, `operator const T*()`, and the explicit
118  conversion template functions `operator U*()`, `operator const U*()` (with
119  any suitable type U). In certain cases, using these methods may have
120  undefined behavior, e.g. if the host pointer is not currently valid. */
121 template <typename T>
122 class Memory
123 {
124 protected:
125  friend class MemoryManager;
126  friend void MemoryPrintFlags(unsigned flags);
127 
128  enum FlagMask: unsigned
129  {
130  REGISTERED = 1 << 0, /**< The host pointer is registered with the
131  MemoryManager */
132  OWNS_HOST = 1 << 1, ///< The host pointer will be deleted by Delete()
133  OWNS_DEVICE = 1 << 2, /**< The device pointer will be deleted by
134  Delete() */
135  OWNS_INTERNAL = 1 << 3, ///< Ownership flag for internal Memory data
136  VALID_HOST = 1 << 4, ///< Host pointer is valid
137  VALID_DEVICE = 1 << 5, ///< %Device pointer is valid
138  USE_DEVICE = 1 << 6, /**< Internal device flag, see e.g.
139  Vector::UseDevice() */
140  ALIAS = 1 << 7 ///< Pointer is an alias
141  };
142 
143  /// Pointer to host memory. Not owned.
144  /** The type of the pointer is given by the field #h_mt; it can be any type
145  from MemoryClass::HOST. */
146  T *h_ptr;
147  int capacity; ///< Size of the allocated memory
148  MemoryType h_mt; ///< Host memory type
149  mutable unsigned flags; ///< Bit flags defined from the #FlagMask enum
150  // 'flags' is mutable so that it can be modified in Set{Host,Device}PtrOwner,
151  // Copy{From,To}, {ReadWrite,Read,Write}.
152 
153 public:
154  /// Default constructor: no initialization.
155  Memory() { }
156 
157  /// Copy constructor: default.
158  Memory(const Memory &orig) = default;
159 
160  /// Move constructor: default.
161  Memory(Memory &&orig) = default;
162 
163  /// Copy-assignment operator: default.
164  Memory &operator=(const Memory &orig) = default;
165 
166  /// Move-assignment operator: default.
167  Memory &operator=(Memory &&orig) = default;
168 
169  /// Allocate host memory for @a size entries.
170  /** The allocation uses the current host memory type returned by
171  MemoryManager::GetHostMemoryType(). */
172  explicit Memory(int size) { New(size); }
173 
174  /** @brief Allocate memory for @a size entries with the given MemoryType
175  @a mt. */
176  /** The newly allocated memory is not initialized, however the given
177  MemoryType is still set as valid. */
178  Memory(int size, MemoryType mt) { New(size, mt); }
179 
180  /** @brief Wrap an externally allocated host pointer, @a ptr with the current
181  host memory type returned by MemoryManager::GetHostMemoryType(). */
182  /** The parameter @a own determines whether @a ptr will be deleted when the
183  method Delete() is called. */
184  explicit Memory(T *ptr, int size, bool own) { Wrap(ptr, size, own); }
185 
186  /// Wrap an externally allocated pointer, @a ptr, of the given MemoryType.
187  /** The new memory object will have the given MemoryType set as valid.
188 
189  The given @a ptr must be allocated appropriately for the given
190  MemoryType.
191 
192  The parameter @a own determines whether @a ptr will be deleted when the
193  method Delete() is called. */
194  Memory(T *ptr, int size, MemoryType mt, bool own)
195  { Wrap(ptr, size, mt, own); }
196 
197  /** @brief Alias constructor. Create a Memory object that points inside the
198  Memory object @a base. */
199  /** The new Memory object uses the same MemoryType(s) as @a base. */
200  Memory(const Memory &base, int offset, int size)
201  { MakeAlias(base, offset, size); }
202 
203  /// Destructor: default.
204  /** @note The destructor will NOT delete the current memory. */
205  ~Memory() = default;
206 
207  /** @brief Return true if the host pointer is owned. Ownership indicates
208  whether the pointer will be deleted by the method Delete(). */
209  bool OwnsHostPtr() const { return flags & OWNS_HOST; }
210 
211  /** @brief Set/clear the ownership flag for the host pointer. Ownership
212  indicates whether the pointer will be deleted by the method Delete(). */
213  void SetHostPtrOwner(bool own) const
214  { flags = own ? (flags | OWNS_HOST) : (flags & ~OWNS_HOST); }
215 
216  /** @brief Return true if the device pointer is owned. Ownership indicates
217  whether the pointer will be deleted by the method Delete(). */
218  bool OwnsDevicePtr() const { return flags & OWNS_DEVICE; }
219 
220  /** @brief Set/clear the ownership flag for the device pointer. Ownership
221  indicates whether the pointer will be deleted by the method Delete(). */
222  void SetDevicePtrOwner(bool own) const
223  { flags = own ? (flags | OWNS_DEVICE) : (flags & ~OWNS_DEVICE); }
224 
225  /** @brief Clear the ownership flags for the host and device pointers, as
226  well as any internal data allocated by the Memory object. */
227  void ClearOwnerFlags() const
229 
230  /// Read the internal device flag.
231  bool UseDevice() const { return flags & USE_DEVICE; }
232 
233  /// Set the internal device flag.
234  void UseDevice(bool use_dev) const
235  { flags = use_dev ? (flags | USE_DEVICE) : (flags & ~USE_DEVICE); }
236 
237  /// Return the size of the allocated memory.
238  int Capacity() const { return capacity; }
239 
240  /// Reset the memory to be empty, ensuring that Delete() will be a no-op.
241  /** This is the Memory class equivalent to setting a pointer to NULL, see
242  Empty().
243 
244  @note The current memory is NOT deleted by this method. */
245  void Reset();
246 
247  /// Reset the memory and set the host memory type.
248  void Reset(MemoryType host_mt);
249 
250  /// Return true if the Memory object is empty, see Reset().
251  /** Default-constructed objects are uninitialized, so they are not guaranteed
252  to be empty. */
253  bool Empty() const { return h_ptr == NULL; }
254 
255  /** @brief Allocate host memory for @a size entries with the current host
256  memory type returned by MemoryManager::GetHostMemoryType(). */
257  /** @note The current memory is NOT deleted by this method. */
258  inline void New(int size);
259 
260  /// Allocate memory for @a size entries with the given MemoryType.
261  /** The newly allocated memory is not initialized, however the given
262  MemoryType is still set as valid.
263 
264  @note The current memory is NOT deleted by this method. */
265  inline void New(int size, MemoryType mt);
266 
267  /** @brief Wrap an externally allocated host pointer, @a ptr with the current
268  host memory type returned by MemoryManager::GetHostMemoryType(). */
269  /** The parameter @a own determines whether @a ptr will be deleted when the
270  method Delete() is called.
271 
272  @note The current memory is NOT deleted by this method. */
273  inline void Wrap(T *ptr, int size, bool own);
274 
275  /// Wrap an externally allocated pointer, @a ptr, of the given MemoryType.
276  /** The new memory object will have the given MemoryType set as valid.
277 
278  The given @a ptr must be allocated appropriately for the given
279  MemoryType.
280 
281  The parameter @a own determines whether @a ptr will be deleted when the
282  method Delete() is called.
283 
284  @note The current memory is NOT deleted by this method. */
285  inline void Wrap(T *ptr, int size, MemoryType mt, bool own);
286 
287  /** Wrap an externally pair of allocated pointers, @a h_ptr and @ d_ptr,
288  of the given host MemoryType @a h_mt. */
289  /** The new memory object will have the device MemoryType set as valid.
290 
291  The given @a h_ptr and @a d_ptr must be allocated appropriately for the
292  given host MemoryType and its associated device MemoryType:
293  - MANAGED => MANAGED,
294  - HOST_DEBUG => DEVICE_DEBUG,
295  - HOST_UMPIRE => DEVICE_UMPIRE,
296  - HOST, HOST_32, HOST_64 => DEVICE.
297 
298  The parameter @a own determines whether both @a h_ptr and @a d_ptr will
299  be deleted when the method Delete() is called.
300 
301  @note Ownership can also be controlled by using the following methods:
302  - ClearOwnerFlags,
303  - SetHostPtrOwner,
304  - SetDevicePtrOwner.
305 
306  @note The current memory is NOT deleted by this method. */
307  inline void Wrap(T *h_ptr, T *d_ptr, int size, MemoryType h_mt, bool own);
308 
309  /// Create a memory object that points inside the memory object @a base.
310  /** The new Memory object uses the same MemoryType(s) as @a base.
311 
312  @note The current memory is NOT deleted by this method. */
313  inline void MakeAlias(const Memory &base, int offset, int size);
314 
315  /** @brief Delete the owned pointers. The Memory is not reset by this method,
316  i.e. it will, generally, not be Empty() after this call. */
317  inline void Delete();
318 
319  /// Array subscript operator for host memory.
320  inline T &operator[](int idx);
321 
322  /// Array subscript operator for host memory, const version.
323  inline const T &operator[](int idx) const;
324 
325  /// Direct access to the host memory as T* (implicit conversion).
326  /** When the type T is const-qualified, this method can be used only if the
327  host pointer is currently valid (the device pointer may be valid or
328  invalid).
329 
330  When the type T is not const-qualified, this method can be used only if
331  the host pointer is the only valid pointer.
332 
333  When the Memory is empty, this method can be used and it returns NULL. */
334  inline operator T*();
335 
336  /// Direct access to the host memory as const T* (implicit conversion).
337  /** This method can be used only if the host pointer is currently valid (the
338  device pointer may be valid or invalid).
339 
340  When the Memory is empty, this method can be used and it returns NULL. */
341  inline operator const T*() const;
342 
343  /// Direct access to the host memory via explicit typecast.
344  /** A pointer to type T must be reinterpret_cast-able to a pointer to type U.
345  In particular, this method cannot be used to cast away const-ness from
346  the base type T.
347 
348  When the type U is const-qualified, this method can be used only if the
349  host pointer is currently valid (the device pointer may be valid or
350  invalid).
351 
352  When the type U is not const-qualified, this method can be used only if
353  the host pointer is the only valid pointer.
354 
355  When the Memory is empty, this method can be used and it returns NULL. */
356  template <typename U>
357  inline explicit operator U*();
358 
359  /// Direct access to the host memory via explicit typecast, const version.
360  /** A pointer to type T must be reinterpret_cast-able to a pointer to type
361  const U.
362 
363  This method can be used only if the host pointer is currently valid (the
364  device pointer may be valid or invalid).
365 
366  When the Memory is empty, this method can be used and it returns NULL. */
367  template <typename U>
368  inline explicit operator const U*() const;
369 
370  /// Get read-write access to the memory with the given MemoryClass.
371  /** If only read or only write access is needed, then the methods
372  Read() or Write() should be used instead of this method.
373 
374  The parameter @a size must not exceed the Capacity(). */
375  inline T *ReadWrite(MemoryClass mc, int size);
376 
377  /// Get read-only access to the memory with the given MemoryClass.
378  /** The parameter @a size must not exceed the Capacity(). */
379  inline const T *Read(MemoryClass mc, int size) const;
380 
381  /// Get write-only access to the memory with the given MemoryClass.
382  /** The parameter @a size must not exceed the Capacity().
383 
384  The contents of the returned pointer is undefined, unless it was
385  validated by a previous call to Read() or ReadWrite() with
386  the same MemoryClass. */
387  inline T *Write(MemoryClass mc, int size);
388 
389  /// Copy the host/device pointer validity flags from @a other to @a *this.
390  /** This method synchronizes the pointer validity flags of two Memory objects
391  that use the same host/device pointers, or when @a *this is an alias
392  (sub-Memory) of @a other. Typically, this method should be called after
393  @a other is manipulated in a way that changes its pointer validity flags
394  (e.g. it was moved from device to host memory). */
395  inline void Sync(const Memory &other) const;
396 
397  /** @brief Update the alias Memory @a *this to match the memory location (all
398  valid locations) of its base Memory, @a base. */
399  /** This method is useful when alias Memory is moved and manipulated in a
400  different memory space. Such operations render the pointer validity flags
401  of the base incorrect. Calling this method will ensure that @a base is
402  up-to-date. Note that this is achieved by moving/copying @a *this (if
403  necessary), and not @a base. */
404  inline void SyncAlias(const Memory &base, int alias_size) const;
405 
406  /** @brief Return a MemoryType that is currently valid. If both the host and
407  the device pointers are currently valid, then the device memory type is
408  returned. */
409  inline MemoryType GetMemoryType() const;
410 
411  /// Copy @a size entries from @a src to @a *this.
412  /** The given @a size should not exceed the Capacity() of the source @a src
413  and the destination, @a *this. */
414  inline void CopyFrom(const Memory &src, int size);
415 
416  /// Copy @a size entries from the host pointer @a src to @a *this.
417  /** The given @a size should not exceed the Capacity() of @a *this. */
418  inline void CopyFromHost(const T *src, int size);
419 
420  /// Copy @a size entries from @a *this to @a dest.
421  /** The given @a size should not exceed the Capacity() of @a *this and the
422  destination, @a dest. */
423  inline void CopyTo(Memory &dest, int size) const
424  { dest.CopyFrom(*this, size); }
425 
426  /// Copy @a size entries from @a *this to the host pointer @a dest.
427  /** The given @a size should not exceed the Capacity() of @a *this. */
428  inline void CopyToHost(T *dest, int size) const;
429 
430  /// Print the internal flags.
431  /** This method can be useful for debugging. It is explicitly instantiated
432  for Memory<T> with T = int and T = double. */
433  inline void PrintFlags() const;
434 
435  /// If both the host and the device data are valid, compare their contents.
436  /** This method can be useful for debugging. It is explicitly instantiated
437  for Memory<T> with T = int and T = double. */
438  inline int CompareHostAndDevice(int size) const;
439 
440 private:
441  // GCC 4.8 workaround: max_align_t is not in std.
442  static constexpr std::size_t def_align_bytes_()
443  {
444  using namespace std;
445  return alignof(max_align_t);
446  }
447  static constexpr std::size_t def_align_bytes = def_align_bytes_();
448  static constexpr std::size_t new_align_bytes =
449  alignof(T) > def_align_bytes ? alignof(T) : def_align_bytes;
450 
451  template <std::size_t align_bytes, bool dummy = true> struct Alloc
452  {
453 #if __cplusplus < 201703L
454  static inline T *New(std::size_t)
455  {
456  // Generate an error in debug mode
457  MFEM_ASSERT(false, "overaligned type cannot use MemoryType::HOST");
458  return nullptr;
459  }
460 #else
461  static inline T *New(std::size_t size) { return new T[size]; }
462 #endif
463  };
464 
465 #if __cplusplus < 201703L
466  template<bool dummy> struct Alloc<def_align_bytes,dummy>
467  {
468  static inline T *New(std::size_t size) { return new T[size]; }
469  };
470 #endif
471 };
472 
473 
474 /** The MFEM memory manager class. Host-side pointers are inserted into this
475  manager which keeps track of the associated device pointer, and where the
476  data currently resides. */
478 {
479 private:
480 
481  typedef MemoryType MemType;
482  typedef Memory<int> Mem;
483 
484  template <typename T> friend class Memory;
485 
486  /// Host memory type set during the Setup.
487  static MemoryType host_mem_type;
488 
489  /// Device memory type set during the Setup.
490  static MemoryType device_mem_type;
491 
492  /// Allow to detect if a global memory manager instance exists.
493  static bool exists;
494 
495  /// Return true if the global memory manager instance exists.
496  static bool Exists() { return exists; }
497 
498  /// Host and device allocator names for Umpire.
499 #ifdef MFEM_USE_UMPIRE
500  static const char *h_umpire_name;
501  static const char *d_umpire_name;
502 #endif
503 
504 private: // Static methods used by the Memory<T> class
505 
506  /// Allocate and register a new pointer. Return the host pointer.
507  /// h_tmp must be already allocated using new T[] if mt is a pure device
508  /// memory type, e.g. CUDA (mt will not be HOST).
509  static void *New_(void *h_tmp, size_t bytes, MemoryType mt, unsigned &flags);
510 
511  /// Register an external pointer of the given MemoryType.
512  /// Return the host pointer.
513  static void *Register_(void *ptr, void *h_ptr, size_t bytes, MemoryType mt,
514  bool own, bool alias, unsigned &flags);
515 
516  /// Register an alias. Note: base_h_ptr may be an alias.
517  static void Alias_(void *base_h_ptr, size_t offset, size_t bytes,
518  unsigned base_flags, unsigned &flags);
519 
520  /// Un-register and free memory identified by its host pointer. Returns the
521  /// memory type of the host pointer.
522  static MemoryType Delete_(void *h_ptr, MemoryType mt, unsigned flags);
523 
524  /// Check if the memory types given the memory class are valid
525  static bool MemoryClassCheck_(MemoryClass mc, void *h_ptr,
526  MemoryType h_mt, size_t bytes, unsigned flags);
527 
528  /// Return the dual memory type of the given one.
529  static MemoryType GetDualMemoryType_(MemoryType mt);
530 
531  /// Return a pointer to the memory identified by the host pointer h_ptr for
532  /// access with the given MemoryClass.
533  static void *ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
534  size_t bytes, unsigned &flags);
535 
536  static const void *Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
537  size_t bytes, unsigned &flags);
538 
539  static void *Write_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
540  size_t bytes, unsigned &flags);
541 
542  static void SyncAlias_(const void *base_h_ptr, void *alias_h_ptr,
543  size_t alias_bytes, unsigned base_flags,
544  unsigned &alias_flags);
545 
546  /// Return the type the of the currently valid memory.
547  /// If more than one types are valid, return a device type.
548  static MemoryType GetDeviceMemoryType_(void *h_ptr);
549 
550  /// Return the type the of the host memory.
551  static MemoryType GetHostMemoryType_(void *h_ptr);
552 
553  /// Verify that h_mt and h_ptr's h_mt (memory or alias) are equal.
554  static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr);
555 
556  /// Copy entries from valid memory type to valid memory type.
557  /// Both dest_h_ptr and src_h_ptr are registered host pointers.
558  static void Copy_(void *dest_h_ptr, const void *src_h_ptr, size_t bytes,
559  unsigned src_flags, unsigned &dest_flags);
560 
561  /// Copy entries from valid memory type to host memory, where dest_h_ptr is
562  /// not a registered host pointer and src_h_ptr is a registered host pointer.
563  static void CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
564  size_t bytes, unsigned src_flags);
565 
566  /// Copy entries from host memory to valid memory type, where dest_h_ptr is a
567  /// registered host pointer and src_h_ptr is not a registered host pointer.
568  static void CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
569  size_t bytes, unsigned &dest_flags);
570 
571  /// Check if the host pointer has been registered in the memory manager.
572  static bool IsKnown_(const void *h_ptr);
573 
574  /** @brief Check if the host pointer has been registered as an alias in the
575  memory manager. */
576  static bool IsAlias_(const void *h_ptr);
577 
578  /// Compare the contents of the host and the device memory.
579  static int CompareHostAndDevice_(void *h_ptr, size_t size, unsigned flags);
580 
581 private:
582 
583  /// Insert a host address @a h_ptr and size *a bytes in the memory map to be
584  /// managed.
585  void Insert(void *h_ptr, size_t bytes, MemoryType h_mt, MemoryType d_mt);
586 
587  /// Insert a device and the host addresses in the memory map
588  void InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
589  MemoryType h_mt, MemoryType d_mt);
590 
591  /// Insert an alias in the alias map
592  void InsertAlias(const void *base_ptr, void *alias_ptr,
593  const size_t bytes, const bool base_is_alias);
594 
595  /// Erase an address from the memory map, as well as all its aliases
596  void Erase(void *h_ptr, bool free_dev_ptr = true);
597 
598  /// Erase an alias from the aliases map
599  void EraseAlias(void *alias_ptr);
600 
601  /// Return the corresponding device pointer of h_ptr,
602  /// allocating and moving the data if needed
603  void *GetDevicePtr(const void *h_ptr, size_t bytes, bool copy_data);
604 
605  /// Return the corresponding device pointer of alias_ptr,
606  /// allocating and moving the data if needed
607  void *GetAliasDevicePtr(const void *alias_ptr, size_t bytes, bool copy_data);
608 
609  /// Return the corresponding host pointer of d_ptr,
610  /// allocating and moving the data if needed
611  void *GetHostPtr(const void *d_ptr, size_t bytes, bool copy_data);
612 
613  /// Return the corresponding host pointer of alias_ptr,
614  /// allocating and moving the data if needed
615  void *GetAliasHostPtr(const void *alias_ptr, size_t bytes, bool copy_data);
616 
617 public:
618  MemoryManager();
619  ~MemoryManager();
620 
621  /// Initialize the memory manager.
622  void Init();
623 
624  /// Configure the Memory manager with given default host and device types
625  /// This method will be called when configuring a device.
626  void Configure(const MemoryType h_mt, const MemoryType d_mt);
627 
628 #ifdef MFEM_USE_UMPIRE
629  /// Set the host and device UMpire allocator names
630  void SetUmpireAllocatorNames(const char *h_name, const char *d_name);
631  const char *GetUmpireAllocatorHostName() { return h_umpire_name; }
632  const char *GetUmpireAllocatorDeviceName() { return d_umpire_name; }
633 #endif
634 
635  /// Free all the device memories
636  void Destroy();
637 
638  /// Return true if the pointer is known by the memory manager
639  bool IsKnown(const void *h_ptr) { return IsKnown_(h_ptr); }
640 
641  /// Return true if the pointer is known by the memory manager as an alias
642  bool IsAlias(const void *h_ptr) { return IsAlias_(h_ptr); }
643 
644  /// Check if the host pointer has been registered in the memory manager
645  void RegisterCheck(void *h_ptr);
646 
647  /// Prints all pointers known by the memory manager,
648  /// returning the number of printed pointers
649  int PrintPtrs(std::ostream &out = mfem::out);
650 
651  /// Prints all aliases known by the memory manager
652  /// returning the number of printed pointers
653  int PrintAliases(std::ostream &out = mfem::out);
654 
655  static MemoryType GetHostMemoryType() { return host_mem_type; }
656  static MemoryType GetDeviceMemoryType() { return device_mem_type; }
657 };
658 
659 
660 // Inline methods
661 
662 template <typename T>
663 inline void Memory<T>::Reset()
664 {
665  h_ptr = NULL;
666  h_mt = MemoryManager::host_mem_type;
667  capacity = 0;
668  flags = 0;
669 }
670 
671 template <typename T>
672 inline void Memory<T>::Reset(MemoryType host_mt)
673 {
674  h_ptr = NULL;
675  h_mt = host_mt;
676  capacity = 0;
677  flags = 0;
678 }
679 
680 template <typename T>
681 inline void Memory<T>::New(int size)
682 {
683  capacity = size;
685  h_mt = MemoryManager::host_mem_type;
686  h_ptr = (h_mt == MemoryType::HOST) ? Alloc<new_align_bytes>::New(size) :
687  (T*)MemoryManager::New_(nullptr, size*sizeof(T), h_mt, flags);
688 }
689 
690 template <typename T>
691 inline void Memory<T>::New(int size, MemoryType mt)
692 {
693  capacity = size;
694  const size_t bytes = size*sizeof(T);
695  const bool mt_host = mt == MemoryType::HOST;
696  if (mt_host) { flags = OWNS_HOST | VALID_HOST; }
697  h_mt = IsHostMemory(mt) ? mt : MemoryManager::GetDualMemoryType_(mt);
698  T *h_tmp = (h_mt == MemoryType::HOST) ?
699  Alloc<new_align_bytes>::New(size) : nullptr;
700  h_ptr = (mt_host) ? h_tmp : (T*)MemoryManager::New_(h_tmp, bytes, mt, flags);
701 }
702 
703 template <typename T>
704 inline void Memory<T>::Wrap(T *ptr, int size, bool own)
705 {
706  h_ptr = ptr;
707  capacity = size;
708  const size_t bytes = size*sizeof(T);
709  flags = (own ? OWNS_HOST : 0) | VALID_HOST;
710  h_mt = MemoryManager::host_mem_type;
711 #ifdef MFEM_DEBUG
712  if (own && MemoryManager::Exists())
713  { MFEM_VERIFY(h_mt == MemoryManager::GetHostMemoryType_(h_ptr),""); }
714 #endif
715  if (own && h_mt != MemoryType::HOST)
716  { MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags); }
717 }
718 
719 template <typename T>
720 inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
721 {
722  capacity = size;
723  if (IsHostMemory(mt))
724  {
725  h_mt = mt;
726  h_ptr = ptr;
727  if (mt == MemoryType::HOST || !own)
728  {
729  // Skip registration
730  flags = (own ? OWNS_HOST : 0) | VALID_HOST;
731  return;
732  }
733  }
734  else
735  {
736  h_mt = MemoryManager::GetDualMemoryType_(mt);
737  h_ptr = (h_mt == MemoryType::HOST) ? new T[size] : nullptr;
738  }
739  flags = 0;
740  h_ptr = (T*)MemoryManager::Register_(ptr, h_ptr, size*sizeof(T), mt,
741  own, false, flags);
742 }
743 
744 template <typename T>
745 inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
746 {
747  h_mt = mt;
748  flags = 0;
749  h_ptr = ptr;
750  capacity = size;
751  MFEM_ASSERT(IsHostMemory(h_mt),"");
752  const size_t bytes = size*sizeof(T);
753  const MemoryType d_mt = MemoryManager::GetDualMemoryType_(h_mt);
754  MemoryManager::Register_(d_ptr, h_ptr, bytes, d_mt, own, false, flags);
755 }
756 
757 template <typename T>
758 inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
759 {
760  capacity = size;
761  h_mt = base.h_mt;
762  h_ptr = base.h_ptr + offset;
763  if (!(base.flags & REGISTERED))
764  { flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE); }
765  else
766  {
767  const size_t s_bytes = size*sizeof(T);
768  const size_t o_bytes = offset*sizeof(T);
769  MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
770  }
771 }
772 
773 template <typename T>
774 inline void Memory<T>::Delete()
775 {
776  const bool registered = flags & REGISTERED;
777  const bool mt_host = h_mt == MemoryType::HOST;
778  const bool std_delete = !registered && mt_host;
779 
780  if (std_delete ||
781  MemoryManager::Delete_((void*)h_ptr, h_mt, flags) == MemoryType::HOST)
782  {
783  if (flags & OWNS_HOST) { delete [] h_ptr; }
784  }
785 }
786 
787 template <typename T>
788 inline T &Memory<T>::operator[](int idx)
789 {
790  MFEM_ASSERT((flags & VALID_HOST) && !(flags & VALID_DEVICE),
791  "invalid host pointer access");
792  return h_ptr[idx];
793 }
794 
795 template <typename T>
796 inline const T &Memory<T>::operator[](int idx) const
797 {
798  MFEM_ASSERT((flags & VALID_HOST), "invalid host pointer access");
799  return h_ptr[idx];
800 }
801 
802 template <typename T>
804 {
805  MFEM_ASSERT(Empty() ||
806  ((flags & VALID_HOST) &&
807  (std::is_const<T>::value || !(flags & VALID_DEVICE))),
808  "invalid host pointer access");
809  return h_ptr;
810 }
811 
812 template <typename T>
813 inline Memory<T>::operator const T*() const
814 {
815  MFEM_ASSERT(Empty() || (flags & VALID_HOST), "invalid host pointer access");
816  return h_ptr;
817 }
818 
819 template <typename T> template <typename U>
821 {
822  MFEM_ASSERT(Empty() ||
823  ((flags & VALID_HOST) &&
824  (std::is_const<U>::value || !(flags & VALID_DEVICE))),
825  "invalid host pointer access");
826  return reinterpret_cast<U*>(h_ptr);
827 }
828 
829 template <typename T> template <typename U>
830 inline Memory<T>::operator const U*() const
831 {
832  MFEM_ASSERT(Empty() || (flags & VALID_HOST), "invalid host pointer access");
833  return reinterpret_cast<U*>(h_ptr);
834 }
835 
836 template <typename T>
837 inline T *Memory<T>::ReadWrite(MemoryClass mc, int size)
838 {
839  const size_t bytes = size * sizeof(T);
840  if (!(flags & REGISTERED))
841  {
842  if (mc == MemoryClass::HOST) { return h_ptr; }
843  MemoryManager::Register_(h_ptr, nullptr, capacity*sizeof(T), h_mt,
845  }
846  return (T*)MemoryManager::ReadWrite_(h_ptr, h_mt, mc, bytes, flags);
847 }
848 
849 template <typename T>
850 inline const T *Memory<T>::Read(MemoryClass mc, int size) const
851 {
852  const size_t bytes = size * sizeof(T);
853  if (!(flags & REGISTERED))
854  {
855  if (mc == MemoryClass::HOST) { return h_ptr; }
856  MemoryManager::Register_(h_ptr, nullptr, capacity*sizeof(T), h_mt,
858  }
859  return (const T*)MemoryManager::Read_(h_ptr, h_mt, mc, bytes, flags);
860 }
861 
862 template <typename T>
863 inline T *Memory<T>::Write(MemoryClass mc, int size)
864 {
865  const size_t bytes = size * sizeof(T);
866  if (!(flags & REGISTERED))
867  {
868  if (mc == MemoryClass::HOST) { return h_ptr; }
869  MemoryManager::Register_(h_ptr, nullptr, capacity*sizeof(T), h_mt,
871  }
872  return (T*)MemoryManager::Write_(h_ptr, h_mt, mc, bytes, flags);
873 }
874 
875 template <typename T>
876 inline void Memory<T>::Sync(const Memory &other) const
877 {
878  if (!(flags & REGISTERED) && (other.flags & REGISTERED))
879  {
880  MFEM_ASSERT(h_ptr == other.h_ptr &&
881  (flags & ALIAS) == (other.flags & ALIAS),
882  "invalid input");
884  }
885  flags = (flags & ~(VALID_HOST | VALID_DEVICE)) |
886  (other.flags & (VALID_HOST | VALID_DEVICE));
887 }
888 
889 template <typename T>
890 inline void Memory<T>::SyncAlias(const Memory &base, int alias_size) const
891 {
892  // Assuming that if *this is registered then base is also registered.
893  MFEM_ASSERT(!(flags & REGISTERED) || (base.flags & REGISTERED),
894  "invalid base state");
895  if (!(base.flags & REGISTERED)) { return; }
896  MemoryManager::SyncAlias_(base.h_ptr, h_ptr, alias_size*sizeof(T),
897  base.flags, flags);
898 }
899 
900 template <typename T>
902 {
903  if (!(flags & VALID_DEVICE)) { return h_mt; }
904  return MemoryManager::GetDeviceMemoryType_(h_ptr);
905 }
906 
907 template <typename T>
908 inline void Memory<T>::CopyFrom(const Memory &src, int size)
909 {
910  if (!(flags & REGISTERED) && !(src.flags & REGISTERED))
911  {
912  if (h_ptr != src.h_ptr && size != 0)
913  {
914  MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
915  "data overlaps!");
916  std::memcpy(h_ptr, src, size*sizeof(T));
917  }
918  // *this is not registered, so (flags & VALID_HOST) must be true
919  }
920  else
921  {
922  MemoryManager::Copy_(h_ptr, src.h_ptr, size*sizeof(T), src.flags, flags);
923  }
924 }
925 
926 template <typename T>
927 inline void Memory<T>::CopyFromHost(const T *src, int size)
928 {
929  if (!(flags & REGISTERED))
930  {
931  if (h_ptr != src && size != 0)
932  {
933  MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
934  "data overlaps!");
935  std::memcpy(h_ptr, src, size*sizeof(T));
936  }
937  // *this is not registered, so (flags & VALID_HOST) must be true
938  }
939  else
940  {
941  MemoryManager::CopyFromHost_(h_ptr, src, size*sizeof(T), flags);
942  }
943 }
944 
945 template <typename T>
946 inline void Memory<T>::CopyToHost(T *dest, int size) const
947 {
948  if (!(flags & REGISTERED))
949  {
950  if (h_ptr != dest && size != 0)
951  {
952  MFEM_ASSERT(h_ptr + size <= dest || dest + size <= h_ptr,
953  "data overlaps!");
954  std::memcpy(dest, h_ptr, size*sizeof(T));
955  }
956  }
957  else
958  {
959  MemoryManager::CopyToHost_(dest, h_ptr, size*sizeof(T), flags);
960  }
961 }
962 
963 
964 /** @brief Print the state of a Memory object based on its internal flags.
965  Useful in a debugger. See also Memory<T>::PrintFlags(). */
966 extern void MemoryPrintFlags(unsigned flags);
967 
968 
969 template <typename T>
970 inline void Memory<T>::PrintFlags() const
971 {
973 }
974 
975 template <typename T>
976 inline int Memory<T>::CompareHostAndDevice(int size) const
977 {
978  if (!(flags & VALID_HOST) || !(flags & VALID_DEVICE)) { return 0; }
979  return MemoryManager::CompareHostAndDevice_(h_ptr, size*sizeof(T), flags);
980 }
981 
982 
983 /// The (single) global memory manager object
984 extern MemoryManager mm;
985 
986 } // namespace mfem
987 
988 #endif // MFEM_MEM_MANAGER_HPP
MemoryType h_mt
Host memory type.
Host memory; aligned at 64 bytes.
bool IsHostMemory(MemoryType mt)
Return true if the given memory type is in MemoryClass::HOST.
Definition: mem_manager.hpp:69
~Memory()=default
Destructor: default.
friend void MemoryPrintFlags(unsigned flags)
Print the state of a Memory object based on its internal flags. Useful in a debugger. See also Memory&lt;T&gt;::PrintFlags().
Device memory; using CUDA or HIP *Malloc and *Free.
const char * GetUmpireAllocatorHostName()
void PrintFlags() const
Print the internal flags.
Device memory; using Umpire.
const char * MemoryTypeName[MemoryTypeSize]
Memory type names, used during Device:: configuration.
void Delete()
Delete the owned pointers. The Memory is not reset by this method, i.e. it will, generally, not be Empty() after this call.
const char * GetUmpireAllocatorDeviceName()
Memory(T *ptr, int size, MemoryType mt, bool own)
Wrap an externally allocated pointer, ptr, of the given MemoryType.
static MemoryType GetHostMemoryType()
void SyncAlias(const Memory &base, int alias_size) const
Update the alias Memory *this to match the memory location (all valid locations) of its base Memory...
Host pointer is valid.
void SetDevicePtrOwner(bool own) const
Set/clear the ownership flag for the device pointer. Ownership indicates whether the pointer will be ...
Host memory; allocated from a &quot;host-debug&quot; pool.
T * Write(MemoryClass mc, int size)
Get write-only access to the memory with the given MemoryClass.
Memory(int size)
Allocate host memory for size entries.
void UseDevice(bool use_dev) const
Set the internal device flag.
void CopyFrom(const Memory &src, int size)
Copy size entries from src to *this.
Number of host and device memory types.
int Capacity() const
Return the size of the allocated memory.
int CompareHostAndDevice(int size) const
If both the host and the device data are valid, compare their contents.
Memory(int size, MemoryType mt)
Allocate memory for size entries with the given MemoryType mt.
MemoryType GetMemoryType() const
Return a MemoryType that is currently valid. If both the host and the device pointers are currently v...
unsigned flags
Bit flags defined from the FlagMask enum.
bool IsAlias(const void *h_ptr)
Return true if the pointer is known by the memory manager as an alias.
Host memory; aligned at 32 bytes.
void Wrap(T *ptr, int size, bool own)
Wrap an externally allocated host pointer, ptr with the current host memory type returned by MemoryMa...
void CopyFromHost(const T *src, int size)
Copy size entries from the host pointer src to *this.
void CopyTo(Memory &dest, int size) const
Copy size entries from *this to dest.
constexpr int DeviceMemoryType
Definition: mem_manager.hpp:48
constexpr int HostMemoryType
Definition: mem_manager.hpp:46
static MemoryType GetDeviceMemoryType()
Memory(const Memory &base, int offset, int size)
Alias constructor. Create a Memory object that points inside the Memory object base.
Ownership flag for internal Memory data.
Device pointer is valid
The host pointer will be deleted by Delete()
bool OwnsHostPtr() const
Return true if the host pointer is owned. Ownership indicates whether the pointer will be deleted by ...
constexpr int MemoryTypeSize
Static casts to &#39;int&#39; and sizes of some useful memory types.
Definition: mem_manager.hpp:45
Memory & operator=(const Memory &orig)=default
Copy-assignment operator: default.
void Reset()
Reset the memory to be empty, ensuring that Delete() will be a no-op.
void ClearOwnerFlags() const
Clear the ownership flags for the host and device pointers, as well as any internal data allocated by...
MemoryType
Memory types supported by MFEM.
Definition: mem_manager.hpp:28
void SetHostPtrOwner(bool own) const
Set/clear the ownership flag for the host pointer. Ownership indicates whether the pointer will be de...
void Sync(const Memory &other) const
Copy the host/device pointer validity flags from other to *this.
bool IsKnown(const void *h_ptr)
Return true if the pointer is known by the memory manager.
A class to initialize the size of a Tensor.
Definition: dtensor.hpp:54
constexpr int HostMemoryTypeSize
Definition: mem_manager.hpp:47
void MakeAlias(const Memory &base, int offset, int size)
Create a memory object that points inside the memory object base.
bool IsDeviceMemory(MemoryType mt)
Definition: mem_manager.hpp:70
Pointer is an alias.
MemoryManager mm
The (single) global memory manager object.
T * h_ptr
Pointer to host memory. Not owned.
int capacity
Size of the allocated memory.
Host memory; using new[] and delete[].
void New(int size)
Allocate host memory for size entries with the current host memory type returned by MemoryManager::Ge...
bool Empty() const
Return true if the Memory object is empty, see Reset().
T * ReadWrite(MemoryClass mc, int size)
Get read-write access to the memory with the given MemoryClass.
friend class MemoryManager
MemoryType GetMemoryType(MemoryClass mc)
Return a suitable MemoryType for a given MemoryClass.
Definition: mem_manager.cpp:51
void CopyToHost(T *dest, int size) const
Copy size entries from *this to the host pointer dest.
Class used by MFEM to store pointers to host and/or device memory.
Memory()
Default constructor: no initialization.
bool OwnsDevicePtr() const
Return true if the device pointer is owned. Ownership indicates whether the pointer will be deleted b...
bool UseDevice() const
Read the internal device flag.
constexpr int DeviceMemoryTypeSize
Definition: mem_manager.hpp:49
Memory(T *ptr, int size, bool own)
Wrap an externally allocated host pointer, ptr with the current host memory type returned by MemoryMa...
Host memory; using Umpire.
T & operator[](int idx)
Array subscript operator for host memory.
OutStream out(std::cout)
Global stream used by the library for standard output. Initially it uses the same std::streambuf as s...
Definition: globals.hpp:66
MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
Return a suitable MemoryClass from a pair of MemoryClasses.
Definition: mem_manager.cpp:99
const T * Read(MemoryClass mc, int size) const
Get read-only access to the memory with the given MemoryClass.
MemoryClass
Memory classes identify sets of memory types.
Definition: mem_manager.hpp:58