MFEM  v4.5.2
Finite element discretization library
device.cpp
Go to the documentation of this file.
1 // Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
2 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
3 // LICENSE and NOTICE for details. LLNL-CODE-806117.
4 //
5 // This file is part of the MFEM library. For more information and source code
6 // availability visit https://mfem.org.
7 //
8 // MFEM is free software; you can redistribute it and/or modify it under the
9 // terms of the BSD-3 license. We welcome feedback and contributions, see file
10 // CONTRIBUTING.md for details.
11 
12 #include "forall.hpp"
13 #include "occa.hpp"
14 #ifdef MFEM_USE_CEED
15 #include "../fem/ceed/interface/util.hpp"
16 #endif
17 
18 #include <unordered_map>
19 #include <string>
20 #include <map>
21 
22 namespace mfem
23 {
24 
25 // Place the following variables in the mfem::internal namespace, so that they
26 // will not be included in the doxygen documentation.
27 namespace internal
28 {
29 
30 #ifdef MFEM_USE_OCCA
31 // Default occa::device used by MFEM.
32 occa::device occaDevice;
33 #endif
34 
35 #ifdef MFEM_USE_CEED
36 Ceed ceed = NULL;
37 
38 ceed::BasisMap ceed_basis_map;
39 ceed::RestrMap ceed_restr_map;
40 #endif
41 
42 // Backends listed by priority, high to low:
43 static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
44 {
49 };
50 
51 // Backend names listed by priority, high to low:
52 static const char *backend_name[Backend::NUM_BACKENDS] =
53 {
54  "ceed-cuda", "occa-cuda", "raja-cuda", "cuda",
55  "ceed-hip", "raja-hip", "hip", "debug",
56  "occa-omp", "raja-omp", "omp",
57  "ceed-cpu", "occa-cpu", "raja-cpu", "cpu"
58 };
59 
60 } // namespace mfem::internal
61 
62 
63 // Initialize the unique global Device variable.
64 Device Device::device_singleton;
65 bool Device::device_env = false;
66 bool Device::mem_host_env = false;
67 bool Device::mem_device_env = false;
68 bool Device::mem_types_set = false;
69 
71 {
72  if (getenv("MFEM_MEMORY") && !mem_host_env && !mem_device_env)
73  {
74  std::string mem_backend(getenv("MFEM_MEMORY"));
75  if (mem_backend == "host")
76  {
77  mem_host_env = true;
78  host_mem_type = MemoryType::HOST;
79  device_mem_type = MemoryType::HOST;
80  }
81  else if (mem_backend == "host32")
82  {
83  mem_host_env = true;
84  host_mem_type = MemoryType::HOST_32;
85  device_mem_type = MemoryType::HOST_32;
86  }
87  else if (mem_backend == "host64")
88  {
89  mem_host_env = true;
90  host_mem_type = MemoryType::HOST_64;
91  device_mem_type = MemoryType::HOST_64;
92  }
93  else if (mem_backend == "umpire")
94  {
95  mem_host_env = true;
96  host_mem_type = MemoryType::HOST_UMPIRE;
97  // Note: device_mem_type will be set to MemoryType::DEVICE_UMPIRE only
98  // when an actual device is configured -- this is done later in
99  // Device::UpdateMemoryTypeAndClass().
100  device_mem_type = MemoryType::HOST_UMPIRE;
101  }
102  else if (mem_backend == "debug")
103  {
104  mem_host_env = true;
105  host_mem_type = MemoryType::HOST_DEBUG;
106  // Note: device_mem_type will be set to MemoryType::DEVICE_DEBUG only
107  // when an actual device is configured -- this is done later in
108  // Device::UpdateMemoryTypeAndClass().
109  device_mem_type = MemoryType::HOST_DEBUG;
110  }
111  else if (false
112 #ifdef MFEM_USE_CUDA
113  || mem_backend == "cuda"
114 #endif
115 #ifdef MFEM_USE_HIP
116  || mem_backend == "hip"
117 #endif
118  )
119  {
120  mem_host_env = true;
121  host_mem_type = MemoryType::HOST;
122  mem_device_env = true;
123  device_mem_type = MemoryType::DEVICE;
124  }
125  else if (mem_backend == "uvm")
126  {
127  mem_host_env = true;
128  mem_device_env = true;
129  host_mem_type = MemoryType::MANAGED;
130  device_mem_type = MemoryType::MANAGED;
131  }
132  else
133  {
134  MFEM_ABORT("Unknown memory backend!");
135  }
136  mm.Configure(host_mem_type, device_mem_type);
137  }
138 
139  if (getenv("MFEM_DEVICE"))
140  {
141  std::string device(getenv("MFEM_DEVICE"));
142  Configure(device);
143  device_env = true;
144  }
145 }
146 
147 
149 {
150  if ( device_env && !destroy_mm) { return; }
151  if (!device_env && destroy_mm && !mem_host_env)
152  {
153  free(device_option);
154 #ifdef MFEM_USE_CEED
155  // Destroy FES -> CeedBasis, CeedElemRestriction hash table contents
156  for (auto entry : internal::ceed_basis_map)
157  {
158  CeedBasisDestroy(&entry.second);
159  }
160  internal::ceed_basis_map.clear();
161  for (auto entry : internal::ceed_restr_map)
162  {
163  CeedElemRestrictionDestroy(&entry.second);
164  }
165  internal::ceed_restr_map.clear();
166  // Destroy Ceed context
167  CeedDestroy(&internal::ceed);
168 #endif
169  mm.Destroy();
170  }
171  Get().ngpu = -1;
172  Get().mode = SEQUENTIAL;
173  Get().backends = Backend::CPU;
174  Get().host_mem_type = MemoryType::HOST;
175  Get().host_mem_class = MemoryClass::HOST;
176  Get().device_mem_type = MemoryType::HOST;
177  Get().device_mem_class = MemoryClass::HOST;
178 }
179 
180 void Device::Configure(const std::string &device, const int device_id)
181 {
182  // If a device was configured via the environment, skip the configuration,
183  // and avoid the 'singleton_device' to destroy the mm.
184  if (device_env)
185  {
186  std::memcpy(this, &Get(), sizeof(Device));
187  Get().destroy_mm = false;
188  return;
189  }
190 
191  std::map<std::string, Backend::Id> bmap;
192  for (int i = 0; i < Backend::NUM_BACKENDS; i++)
193  {
194  bmap[internal::backend_name[i]] = internal::backend_list[i];
195  }
196  std::string::size_type beg = 0, end, option;
197  while (1)
198  {
199  end = device.find(',', beg);
200  end = (end != std::string::npos) ? end : device.size();
201  const std::string bname = device.substr(beg, end - beg);
202  option = bname.find(':');
203  if (option==std::string::npos) // No option
204  {
205  const std::string backend = bname;
206  std::map<std::string, Backend::Id>::iterator it = bmap.find(backend);
207  MFEM_VERIFY(it != bmap.end(), "invalid backend name: '" << backend << '\'');
208  Get().MarkBackend(it->second);
209  }
210  else
211  {
212  const std::string backend = bname.substr(0, option);
213  const std::string boption = bname.substr(option+1);
214  Get().device_option = strdup(boption.c_str());
215  std::map<std::string, Backend::Id>::iterator it = bmap.find(backend);
216  MFEM_VERIFY(it != bmap.end(), "invalid backend name: '" << backend << '\'');
217  Get().MarkBackend(it->second);
218  }
219  if (end == device.size()) { break; }
220  beg = end + 1;
221  }
222 
223  // OCCA_CUDA and CEED_CUDA need CUDA or RAJA_CUDA:
226  {
227  Get().MarkBackend(Backend::CUDA);
228  }
229  // CEED_HIP needs HIP:
231  {
232  Get().MarkBackend(Backend::HIP);
233  }
234  // OCCA_OMP will use OMP or RAJA_OMP unless MFEM_USE_OPENMP=NO:
235 #ifdef MFEM_USE_OPENMP
237  {
238  Get().MarkBackend(Backend::OMP);
239  }
240 #endif
241 
242  // Perform setup.
243  Get().Setup(device_id);
244 
245  // Enable the device
246  Enable();
247 
248  // Copy all data members from the global 'singleton_device' into '*this'.
249  if (this != &Get()) { std::memcpy(this, &Get(), sizeof(Device)); }
250 
251  // Only '*this' will call the MemoryManager::Destroy() method.
252  destroy_mm = true;
253 }
254 
255 // static method
257 {
258  // If the device and/or the MemoryTypes are configured through the
259  // environment (variables 'MFEM_DEVICE', 'MFEM_MEMORY'), ignore calls to this
260  // method.
261  if (mem_host_env || mem_device_env || device_env) { return; }
262 
263  MFEM_VERIFY(!IsConfigured(), "the default MemoryTypes can only be set before"
264  " Device construction and configuration");
265  MFEM_VERIFY(IsHostMemory(h_mt),
266  "invalid host MemoryType, h_mt = " << (int)h_mt);
267  MFEM_VERIFY(IsDeviceMemory(d_mt) || d_mt == h_mt,
268  "invalid device MemoryType, d_mt = " << (int)d_mt
269  << " (h_mt = " << (int)h_mt << ')');
270 
271  Get().host_mem_type = h_mt;
272  Get().device_mem_type = d_mt;
273  mem_types_set = true;
274 
275  // h_mt and d_mt will be set as dual to each other during configuration by
276  // the call mm.Configure(...) in UpdateMemoryTypeAndClass()
277 }
278 
279 void Device::Print(std::ostream &os)
280 {
281  os << "Device configuration: ";
282  bool add_comma = false;
283  for (int i = 0; i < Backend::NUM_BACKENDS; i++)
284  {
285  if (backends & internal::backend_list[i])
286  {
287  if (add_comma) { os << ','; }
288  add_comma = true;
289  os << internal::backend_name[i];
290  }
291  }
292  os << '\n';
293 #ifdef MFEM_USE_CEED
295  {
296  const char *ceed_backend;
297  CeedGetResource(internal::ceed, &ceed_backend);
298  os << "libCEED backend: " << ceed_backend << '\n';
299  }
300 #endif
301  os << "Memory configuration: "
302  << MemoryTypeName[static_cast<int>(host_mem_type)];
304  {
305  os << ',' << MemoryTypeName[static_cast<int>(device_mem_type)];
306  }
307  os << std::endl;
308 }
309 
310 void Device::UpdateMemoryTypeAndClass()
311 {
312  const bool debug = Device::Allows(Backend::DEBUG_DEVICE);
313 
314  const bool device = Device::Allows(Backend::DEVICE_MASK);
315 
316 #ifdef MFEM_USE_UMPIRE
317  // If MFEM has been compiled with Umpire support, use it as the default
318  if (!mem_host_env && !mem_types_set)
319  {
320  host_mem_type = MemoryType::HOST_UMPIRE;
321  if (!mem_device_env)
322  {
323  device_mem_type = MemoryType::HOST_UMPIRE;
324  }
325  }
326 #endif
327 
328  // Enable the device memory type
329  if (device)
330  {
331  if (!mem_device_env)
332  {
333  if (mem_host_env)
334  {
335  switch (host_mem_type)
336  {
338  device_mem_type = MemoryType::DEVICE_UMPIRE;
339  break;
341  device_mem_type = MemoryType::DEVICE_DEBUG;
342  break;
343  default:
344  device_mem_type = MemoryType::DEVICE;
345  }
346  }
347  else if (!mem_types_set)
348  {
349 #ifndef MFEM_USE_UMPIRE
350  device_mem_type = MemoryType::DEVICE;
351 #else
352  device_mem_type = MemoryType::DEVICE_UMPIRE;
353 #endif
354  }
355  }
356  device_mem_class = MemoryClass::DEVICE;
357  }
358 
359  // Enable the UVM shortcut when requested
360  if (device && device_option && !strcmp(device_option, "uvm"))
361  {
362  host_mem_type = MemoryType::MANAGED;
363  device_mem_type = MemoryType::MANAGED;
364  }
365 
366  // Enable the DEBUG mode when requested
367  if (debug)
368  {
369  host_mem_type = MemoryType::HOST_DEBUG;
370  device_mem_type = MemoryType::DEVICE_DEBUG;
371  }
372 
373  MFEM_VERIFY(!device || IsDeviceMemory(device_mem_type),
374  "invalid device memory configuration!");
375 
376  // Update the memory manager with the new settings
377  mm.Configure(host_mem_type, device_mem_type);
378 }
379 
380 void Device::Enable()
381 {
382  const bool accelerated = Get().backends & ~(Backend::CPU);
383  if (accelerated) { Get().mode = Device::ACCELERATED;}
384  Get().UpdateMemoryTypeAndClass();
385 }
386 
387 #ifdef MFEM_USE_CUDA
388 static void DeviceSetup(const int dev, int &ngpu)
389 {
390  ngpu = CuGetDeviceCount();
391  MFEM_VERIFY(ngpu > 0, "No CUDA device found!");
392  MFEM_GPU_CHECK(cudaSetDevice(dev));
393 }
394 #endif
395 
396 static void CudaDeviceSetup(const int dev, int &ngpu)
397 {
398 #ifdef MFEM_USE_CUDA
399  DeviceSetup(dev, ngpu);
400 #else
401  MFEM_CONTRACT_VAR(dev);
402  MFEM_CONTRACT_VAR(ngpu);
403 #endif
404 }
405 
406 static void HipDeviceSetup(const int dev, int &ngpu)
407 {
408 #ifdef MFEM_USE_HIP
409  MFEM_GPU_CHECK(hipGetDeviceCount(&ngpu));
410  MFEM_VERIFY(ngpu > 0, "No HIP device found!");
411  MFEM_GPU_CHECK(hipSetDevice(dev));
412 #else
413  MFEM_CONTRACT_VAR(dev);
414  MFEM_CONTRACT_VAR(ngpu);
415 #endif
416 }
417 
418 static void RajaDeviceSetup(const int dev, int &ngpu)
419 {
420 #ifdef MFEM_USE_CUDA
421  if (ngpu <= 0) { DeviceSetup(dev, ngpu); }
422 #elif defined(MFEM_USE_HIP)
423  HipDeviceSetup(dev, ngpu);
424 #else
425  MFEM_CONTRACT_VAR(dev);
426  MFEM_CONTRACT_VAR(ngpu);
427 #endif
428 }
429 
430 static void OccaDeviceSetup(const int dev)
431 {
432 #ifdef MFEM_USE_OCCA
433  const int cpu = Device::Allows(Backend::OCCA_CPU);
434  const int omp = Device::Allows(Backend::OCCA_OMP);
435  const int cuda = Device::Allows(Backend::OCCA_CUDA);
436  if (cpu + omp + cuda > 1)
437  {
438  MFEM_ABORT("Only one OCCA backend can be configured at a time!");
439  }
440  if (cuda)
441  {
442 #if OCCA_CUDA_ENABLED
443  std::string mode("mode: 'CUDA', device_id : ");
444  internal::occaDevice.setup(mode.append(1,'0'+dev));
445 #else
446  MFEM_ABORT("the OCCA CUDA backend requires OCCA built with CUDA!");
447 #endif
448  }
449  else if (omp)
450  {
451 #if OCCA_OPENMP_ENABLED
452  internal::occaDevice.setup("mode: 'OpenMP'");
453 #else
454  MFEM_ABORT("the OCCA OpenMP backend requires OCCA built with OpenMP!");
455 #endif
456  }
457  else
458  {
459  internal::occaDevice.setup("mode: 'Serial'");
460  }
461 
462  std::string mfemDir;
463  if (occa::io::exists(MFEM_INSTALL_DIR "/include/mfem/"))
464  {
465  mfemDir = MFEM_INSTALL_DIR "/include/mfem/";
466  }
467  else if (occa::io::exists(MFEM_SOURCE_DIR))
468  {
469  mfemDir = MFEM_SOURCE_DIR;
470  }
471  else
472  {
473  MFEM_ABORT("Cannot find OCCA kernels in MFEM_INSTALL_DIR or MFEM_SOURCE_DIR");
474  }
475 
476  occa::io::addLibraryPath("mfem", mfemDir);
477  occa::loadKernels("mfem");
478 #else
479  MFEM_CONTRACT_VAR(dev);
480  MFEM_ABORT("the OCCA backends require MFEM built with MFEM_USE_OCCA=YES");
481 #endif
482 }
483 
484 static void CeedDeviceSetup(const char* ceed_spec)
485 {
486 #ifdef MFEM_USE_CEED
487  CeedInit(ceed_spec, &internal::ceed);
488  const char *ceed_backend;
489  CeedGetResource(internal::ceed, &ceed_backend);
490  if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self") &&
491  strcmp(ceed_spec, "/gpu/hip"))
492  {
493  mfem::out << std::endl << "WARNING!!!\n"
494  "libCEED is not using the requested backend!!!\n"
495  "WARNING!!!\n" << std::endl;
496  }
497 #ifdef MFEM_DEBUG
498  CeedSetErrorHandler(internal::ceed, CeedErrorStore);
499 #endif
500 #else
501  MFEM_CONTRACT_VAR(ceed_spec);
502 #endif
503 }
504 
505 void Device::Setup(const int device_id)
506 {
507  MFEM_VERIFY(ngpu == -1, "the mfem::Device is already configured!");
508 
509  ngpu = 0;
510  dev = device_id;
511 #ifndef MFEM_USE_CUDA
512  MFEM_VERIFY(!Allows(Backend::CUDA_MASK),
513  "the CUDA backends require MFEM built with MFEM_USE_CUDA=YES");
514 #endif
515 #ifndef MFEM_USE_HIP
516  MFEM_VERIFY(!Allows(Backend::HIP_MASK),
517  "the HIP backends require MFEM built with MFEM_USE_HIP=YES");
518 #endif
519 #ifndef MFEM_USE_RAJA
520  MFEM_VERIFY(!Allows(Backend::RAJA_MASK),
521  "the RAJA backends require MFEM built with MFEM_USE_RAJA=YES");
522 #endif
523 #ifndef MFEM_USE_OPENMP
524  MFEM_VERIFY(!Allows(Backend::OMP|Backend::RAJA_OMP),
525  "the OpenMP and RAJA OpenMP backends require MFEM built with"
526  " MFEM_USE_OPENMP=YES");
527 #endif
528 #ifndef MFEM_USE_CEED
529  MFEM_VERIFY(!Allows(Backend::CEED_MASK),
530  "the CEED backends require MFEM built with MFEM_USE_CEED=YES");
531 #else
532  int ceed_cpu = Allows(Backend::CEED_CPU);
533  int ceed_cuda = Allows(Backend::CEED_CUDA);
534  int ceed_hip = Allows(Backend::CEED_HIP);
535  MFEM_VERIFY(ceed_cpu + ceed_cuda + ceed_hip <= 1,
536  "Only one CEED backend can be enabled at a time!");
537 #endif
538  if (Allows(Backend::CUDA)) { CudaDeviceSetup(dev, ngpu); }
539  if (Allows(Backend::HIP)) { HipDeviceSetup(dev, ngpu); }
541  { RajaDeviceSetup(dev, ngpu); }
542  // The check for MFEM_USE_OCCA is in the function OccaDeviceSetup().
543  if (Allows(Backend::OCCA_MASK)) { OccaDeviceSetup(dev); }
545  {
546  if (!device_option)
547  {
548  CeedDeviceSetup("/cpu/self");
549  }
550  else
551  {
552  CeedDeviceSetup(device_option);
553  }
554  }
556  {
557  if (!device_option)
558  {
559  // NOTE: libCEED's /gpu/cuda/gen backend is non-deterministic!
560  CeedDeviceSetup("/gpu/cuda/gen");
561  }
562  else
563  {
564  CeedDeviceSetup(device_option);
565  }
566  }
568  {
569  if (!device_option)
570  {
571  CeedDeviceSetup("/gpu/hip");
572  }
573  else
574  {
575  CeedDeviceSetup(device_option);
576  }
577  }
578  if (Allows(Backend::DEBUG_DEVICE)) { ngpu = 1; }
579 }
580 
581 } // mfem
std::unordered_map< const RestrKey, CeedElemRestriction, RestrHash > RestrMap
Definition: util.hpp:148
static bool IsConfigured()
Return true if Configure() has been called previously.
Definition: device.hpp:240
[device] OCCA CUDA backend. Enabled when MFEM_USE_OCCA = YES and MFEM_USE_CUDA = YES.
Definition: device.hpp:59
Host memory; aligned at 64 bytes.
bool IsHostMemory(MemoryType mt)
Return true if the given memory type is in MemoryClass::HOST.
Definition: mem_manager.hpp:85
[host] OCCA OpenMP backend. Enabled when MFEM_USE_OCCA = YES.
Definition: device.hpp:56
~Device()
Destructor.
Definition: device.cpp:148
Device memory; using CUDA or HIP *Malloc and *Free.
[device] CEED CUDA backend working together with the CUDA backend. Enabled when MFEM_USE_CEED = YES a...
Definition: device.hpp:66
[host] RAJA OpenMP backend. Enabled when MFEM_USE_RAJA = YES and MFEM_USE_OPENMP = YES...
Definition: device.hpp:45
Biwise-OR of all HIP backends.
Definition: device.hpp:90
Host memory; allocated from a "host-debug" pool.
MemoryManager mm
The (single) global memory manager object.
void Configure(const MemoryType h_mt, const MemoryType d_mt)
Configure the Memory manager with given default host and device types. This method will be called whe...
void Print(std::ostream &out=mfem::out)
Print the configuration of the MFEM virtual device object.
Definition: device.cpp:279
[device] RAJA CUDA backend. Enabled when MFEM_USE_RAJA = YES and MFEM_USE_CUDA = YES.
Definition: device.hpp:48
int CuGetDeviceCount()
Get the number of CUDA devices.
Definition: cuda.cpp:185
void Configure(const std::string &device, const int dev=0)
Configure the Device backends.
Definition: device.cpp:180
Host memory; aligned at 32 bytes.
Device()
Default constructor. Unless Configure() is called later, the default Backend::CPU will be used...
Definition: device.cpp:70
const char * MemoryTypeName[MemoryTypeSize]
Memory type names, used during Device:: configuration.
Id
In the documentation below, we use square brackets to indicate the type of the backend: host or devic...
Definition: device.hpp:30
[host] OCCA CPU backend: sequential execution on each MPI rank. Enabled when MFEM_USE_OCCA = YES...
Definition: device.hpp:54
Number of backends: from (1 << 0) to (1 << (NUM_BACKENDS-1)).
Definition: device.hpp:83
static void SetMemoryTypes(MemoryType h_mt, MemoryType d_mt)
Set the default host and device MemoryTypes, h_mt and d_mt.
Definition: device.cpp:256
void Destroy()
Free all the device memories.
[host] RAJA CPU backend: sequential execution on each MPI rank. Enabled when MFEM_USE_RAJA = YES...
Definition: device.hpp:42
std::unordered_map< const BasisKey, CeedBasis, BasisHash > BasisMap
Definition: util.hpp:126
[host] Default CPU backend: sequential execution on each MPI rank.
Definition: device.hpp:33
Biwise-OR of all CUDA backends.
Definition: device.hpp:88
OutStream out(std::cout)
Global stream used by the library for standard output. Initially it uses the same std::streambuf as s...
Definition: globals.hpp:66
MemoryType
Memory types supported by MFEM.
Definition: mem_manager.hpp:31
[host] CEED CPU backend. GPU backends can still be used, but with expensive memory transfers...
Definition: device.hpp:62
[host] OpenMP backend. Enabled when MFEM_USE_OPENMP = YES.
Definition: device.hpp:35
static bool Allows(unsigned long b_mask)
Return true if any of the backends in the backend mask, b_mask, are allowed.
Definition: device.hpp:258
bool IsDeviceMemory(MemoryType mt)
Return true if the given memory type is in MemoryClass::DEVICE.
Definition: mem_manager.hpp:88
Host memory; using new[] and delete[].
[device] CEED HIP backend working together with the HIP backend. Enabled when MFEM_USE_CEED = YES and...
Definition: device.hpp:69
Biwise-OR of all OCCA backends.
Definition: device.hpp:101
Biwise-OR of all RAJA backends.
Definition: device.hpp:99
[device] RAJA HIP backend. Enabled when MFEM_USE_RAJA = YES and MFEM_USE_HIP = YES.
Definition: device.hpp:51
Biwise-OR of all device backends.
Definition: device.hpp:96
The MFEM Device class abstracts hardware devices such as GPUs, as well as programming models such as ...
Definition: device.hpp:121
Bitwise-OR of all CEED backends.
Definition: device.hpp:94
[device] HIP backend. Enabled when MFEM_USE_HIP = YES.
Definition: device.hpp:39
[device] CUDA backend. Enabled when MFEM_USE_CUDA = YES.
Definition: device.hpp:37
[device] Debug backend: host memory is READ/WRITE protected while a device is in use. It allows to test the "device" code-path (using separate host/device memory pools and host <-> device transfers) without any GPU hardware. As &#39;DEBUG&#39; is sometimes used as a macro, _DEVICE has been added to avoid conflicts.
Definition: device.hpp:75