MFEM  v4.5.1
Finite element discretization library
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
schwarz_ex1p.cpp
Go to the documentation of this file.
1 // Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
2 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
3 // LICENSE and NOTICE for details. LLNL-CODE-806117.
4 //
5 // This file is part of the MFEM library. For more information and source code
6 // availability visit https://mfem.org.
7 //
8 // MFEM is free software; you can redistribute it and/or modify it under the
9 // terms of the BSD-3 license. We welcome feedback and contributions, see file
10 // CONTRIBUTING.md for details.
11 //
12 // -------------------------------------------------
13 // Overlapping Grids Miniapp: Poisson problem (ex1p)
14 // -------------------------------------------------
15 //
16 // This example code demonstrates use of MFEM to solve the Poisson problem:
17 //
18 // -Delta u = 1 \in [0, 1]^2, u_b = 0 \in \dO
19 //
20 // on two overlapping grids. Using simultaneous Schwarz iterations, the Poisson
21 // equation is solved iteratively, with boundary data interpolated between the
22 // overlapping boundaries for each grid. The overlapping Schwarz method was
23 // introduced by H. A. Schwarz in 1870, see also Section 2.2 of "Stability
24 // analysis of a singlerate and multirate predictor-corrector scheme for
25 // overlapping grids" by Mittal, Dutta and Fischer, arXiv:2010.00118.
26 //
27 // Compile with: make schwarz_ex1p
28 //
29 // Sample runs: mpirun -np 4 schwarz_ex1p -nm 3 -np1 2 -np2 1 -np3 1
30 // mpirun -np 4 schwarz_ex1p -nm 2 -np1 2 -np2 2
31 // mpirun -np 4 schwarz_ex1p -nm 2 -np1 2 -np2 2 -m1 ../../data/star.mesh -m2 ../../data/beam-quad.mesh
32 
33 #include "mfem.hpp"
34 #include <fstream>
35 #include <iostream>
36 
37 using namespace std;
38 using namespace mfem;
39 
40 // Method to use FindPointsGSLIB to determine the boundary points of a mesh that
41 // are interior to another mesh.
43  Vector &vxyz, int color,
44  Array<int> ess_tdof_list,
45  Array<int> &ess_tdof_list_int, int dim)
46 {
47  int number_boundary = ess_tdof_list.Size(),
48  number_true = vxyz.Size()/dim;
49 
50  Vector bnd(number_boundary*dim);
51  Array<unsigned int> colorv(number_boundary);
52  for (int i = 0; i < number_boundary; i++)
53  {
54  int idx = ess_tdof_list[i];
55  for (int d = 0; d < dim; d++)
56  {
57  bnd(i+d*number_boundary) = vxyz(idx + d*number_true);
58  }
59  colorv[i] = (unsigned int)color;
60  }
61 
62  finder.FindPoints(bnd, colorv);
63 
64  const Array<unsigned int> &code_out = finder.GetCode();
65 
66  // Setup ess_tdof_list_int
67  for (int i = 0; i < number_boundary; i++)
68  {
69  int idx = ess_tdof_list[i];
70  if (code_out[i] != 2) { ess_tdof_list_int.Append(idx); }
71  }
72 }
73 
74 int main(int argc, char *argv[])
75 {
76  // Initialize MPI and HYPRE.
77  Mpi::Init(argc, argv);
78  int num_procs = Mpi::WorldSize();
79  int myid = Mpi::WorldRank();
80  Hypre::Init();
81 
82  // Parse command-line options.
83  int lim_meshes = 3; // should be greater than nmeshes
84  Array <const char *> mesh_file_list(lim_meshes);
85  Array <int> np_list(lim_meshes), rs_levels(lim_meshes),
86  rp_levels(lim_meshes);
87  mesh_file_list[0] = "../../data/square-disc.mesh";
88  mesh_file_list[1] = "../../data/inline-quad.mesh";
89  mesh_file_list[2] = "../../data/inline-quad.mesh";
90  int order = 2;
91  bool visualization = true;
92  rs_levels = 0;
93  rp_levels = 0;
94  np_list = 0;
95  double rel_tol = 1.e-8;
96  int nmeshes = 3;
97 
98  OptionsParser args(argc, argv);
99  args.AddOption(&mesh_file_list[0], "-m1", "--mesh",
100  "Mesh file to use.");
101  args.AddOption(&mesh_file_list[1], "-m2", "--mesh",
102  "Mesh file to use.");
103  args.AddOption(&mesh_file_list[2], "-m3", "--mesh",
104  "Mesh file to use.");
105  args.AddOption(&order, "-o", "--order",
106  "Finite element order (polynomial degree) or -1 for"
107  " isoparametric space.");
108  args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
109  "--no-visualization",
110  "Enable or disable GLVis visualization.");
111  args.AddOption(&rs_levels[0], "-r1", "--refine-serial",
112  "Number of times to refine the mesh 1 uniformly in serial.");
113  args.AddOption(&rs_levels[1], "-r2", "--refine-serial",
114  "Number of times to refine the mesh 2 uniformly in serial.");
115  args.AddOption(&rs_levels[2], "-r3", "--refine-serial",
116  "Number of times to refine the mesh 3 uniformly in serial.");
117  args.AddOption(&rp_levels[0], "-rp1", "--refine-parallel",
118  "Number of times to refine the mesh 1 uniformly in parallel.");
119  args.AddOption(&rp_levels[1], "-rp2", "--refine-parallel",
120  "Number of times to refine the mesh 2 uniformly in parallel.");
121  args.AddOption(&rp_levels[2], "-rp3", "--refine-parallel",
122  "Number of times to refine the mesh 3 uniformly in parallel.");
123  args.AddOption(&np_list[0], "-np1", "--np1",
124  "number of MPI ranks for mesh 1");
125  args.AddOption(&np_list[1], "-np2", "--np2",
126  "number of MPI ranks for mesh 2");
127  args.AddOption(&np_list[2], "-np3", "--np3",
128  "number of MPI ranks for mesh 3");
129  args.AddOption(&nmeshes, "-nm", "--nm",
130  "number of meshes");
131  args.AddOption(&rel_tol, "-rt", "--relative tolerance",
132  "Tolerance for Schwarz iteration convergence criterion.");
133 
134  args.Parse();
135  if (!args.Good())
136  {
137  args.PrintUsage(cout);
138  return 1;
139  }
140  if (myid == 0)
141  {
142  args.PrintOptions(cout);
143  }
144 
145  // Check number of mpi ranks specified for each mesh. If individual mpi ranks
146  // are not specified for all the meshes, set some default values.
147  MFEM_VERIFY(num_procs >= nmeshes, "Not enough MPI ranks.");
148  if (np_list.Sum() == 0)
149  {
150  int np_per_mesh = num_procs/nmeshes;
151  for (int i = 0; i < nmeshes; i++)
152  {
153  np_list[i] = np_per_mesh;
154  }
155  np_list[nmeshes-1] += num_procs - nmeshes*np_per_mesh;
156  }
157  MFEM_VERIFY(np_list.Sum() == num_procs, " The individual mpi ranks for each"
158  " of the meshes do not add up to the total ranks specified.");
159 
160  // Setup MPI communicator for each mesh by splitting MPI_COMM_WORLD.
161  MPI_Comm *comml = new MPI_Comm;
162  int color = 0;
163  int npsum = 0;
164  for (int i = 0; i < nmeshes; i++)
165  {
166  npsum += np_list[i];
167  if (myid < npsum) { color = i; break; }
168  }
169 
170  MPI_Comm_split(MPI_COMM_WORLD, color, myid, comml);
171  int myidlocal, numproclocal;
172  MPI_Comm_rank(*comml, &myidlocal);
173  MPI_Comm_size(*comml, &numproclocal);
174 
175  // Read the mesh from the given mesh file. We can handle triangular,
176  // quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
177  // the same code.
178  Mesh *mesh = new Mesh(mesh_file_list[color], 1, 1);
179  int dim = mesh->Dimension();
180 
181  // Refine the mesh to increase the resolution. In this example we do
182  // 'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
183  // largest number that gives a final mesh with no more than 50,000 elements.
184  for (int lev = 0; lev < rs_levels[color]; lev++) { mesh->UniformRefinement(); }
185 
186  // Define a parallel mesh by a partitioning of the serial mesh. Refine this
187  // mesh further in parallel to increase the resolution. Once the parallel
188  // mesh is defined, the serial mesh can be deleted.
189  ParMesh *pmesh;
190  pmesh = new ParMesh(*comml, *mesh);
191  for (int l = 0; l < rp_levels[color]; l++)
192  {
193  pmesh->UniformRefinement();
194  }
195  delete mesh;
196 
197  // Define a finite element space on the mesh. Here we use continuous
198  // Lagrange finite elements of the specified order. If order < 1, we
199  // instead use an isoparametric/isogeometric space.
201  if (order > 0)
202  {
203  fec = new H1_FECollection(order, dim);
204  }
205  else if (pmesh->GetNodes())
206  {
207  fec = pmesh->GetNodes()->OwnFEC();
208  if (myid == 0)
209  {
210  cout << "Using isoparametric FEs: " << fec->Name() << endl;
211  }
212  }
213  else
214  {
215  fec = new H1_FECollection(order = 1, dim);
216  }
217  ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
218  HYPRE_BigInt size = fespace->GlobalTrueVSize();
219  if (myid == 0)
220  {
221  cout << "Number of finite element unknowns: " << size << endl;
222  }
223 
224  // Determine the list of true (i.e. conforming) essential boundary dofs.
225  // In this example, the boundary conditions are defined by marking all
226  // the boundary attributes from the mesh as essential (Dirichlet) and
227  // converting them to a list of true dofs.
228  Array<int> ess_tdof_list;
229  if (pmesh->bdr_attributes.Size())
230  {
231  Array<int> ess_bdr(pmesh->bdr_attributes.Max());
232  ess_bdr = 1;
233  fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
234  }
235 
236  // Set up the linear form b(.) which corresponds to the right-hand side of
237  // the FEM linear system, which in this case is (1,phi_i) where phi_i are
238  // the basis functions in the finite element fespace1.
239  ParLinearForm *b = new ParLinearForm(fespace);
240  ConstantCoefficient one(1.0);
242  b->Assemble();
243 
244  // Define the solution vector x as a finite element grid function
245  // corresponding to fespace1. Initialize x with initial guess of zero,
246  // which satisfies the boundary conditions.
247  ParGridFunction x(fespace);
248  x = 0.0;
249  x.SetTrueVector();
250 
251  // Setup FindPointsGSLIB and determine points on each mesh's boundary that
252  // are interior to another mesh.
253  pmesh->SetCurvature(order, false, dim, Ordering::byNODES);
254  {
255  Vector vxyz = *pmesh->GetNodes();
256 
257  // For the default mesh inputs, we need to rescale inline-quad.mesh such
258  // that it does not cover the entire domain [0, 1]^2 and still has a
259  // non-trivial overlap with the other mesh.
260  if (strcmp(mesh_file_list[0], "../../data/square-disc.mesh") == 0 &&
261  strcmp(mesh_file_list[1], "../../data/inline-quad.mesh") == 0 &&
262  strcmp(mesh_file_list[2], "../../data/inline-quad.mesh") == 0 )
263  {
264  if (nmeshes == 2)
265  {
266  if (color == 1) // rescale from [0, 1]^2 to [0.25, 0.75]^2
267  {
268  for (int i = 0; i < vxyz.Size(); i++)
269  {
270  vxyz(i) = 0.5 + 0.5*(vxyz(i)-0.5);
271  }
272  }
273  }
274  else if (nmeshes == 3)
275  {
276  if (color == 1)
277  {
278  // rescale from [0, 1]^2 to [0.21, 0.61] in x and [0.25, 0.75] in y
279  const int pts_cnt = vxyz.Size()/dim;
280  for (int i = 0; i < pts_cnt; i++)
281  {
282  vxyz(i) = 0.41 + 0.4*(vxyz(i)-0.5);
283  }
284  for (int i = 0; i < pts_cnt; i++)
285  {
286  vxyz(i+pts_cnt) = 0.5 + 0.5*(vxyz(i+pts_cnt)-0.5);
287  }
288  }
289  else if (color == 2)
290  {
291  // rescale from [0, 1]^2 to [0.4, 0.8] in x and [0.2, 0.8] in y
292  const int pts_cnt = vxyz.Size()/dim;
293  for (int i = 0; i < pts_cnt; i++)
294  {
295  vxyz(i) = 0.6 + 0.4*(vxyz(i)-0.5);
296  }
297  for (int i = 0; i < pts_cnt; i++)
298  {
299  vxyz(i+pts_cnt) = 0.5 + 0.6*(vxyz(i+pts_cnt)-0.5);
300  }
301  }
302  }
303  }
304  pmesh->SetNodes(vxyz);
305  }
306 
307  pmesh->GetNodes()->SetTrueVector();
308  Vector vxyz = pmesh->GetNodes()->GetTrueVector();
309 
310 
311  OversetFindPointsGSLIB finder(MPI_COMM_WORLD);
312  finder.Setup(*pmesh, color);
313 
314  Array<int> ess_tdof_list_int;
315  GetInterdomainBoundaryPoints(finder, vxyz, color,
316  ess_tdof_list, ess_tdof_list_int, dim);
317 
318  // Use FindPointsGSLIB to interpolate the solution at interdomain boundary
319  // points.
320  const int number_boundary = ess_tdof_list_int.Size(),
321  number_true = vxyz.Size()/dim;
322 
323  int number_boundary_g = number_boundary;
324  MPI_Allreduce(&number_boundary, &number_boundary_g, 1, MPI_INT, MPI_SUM,
325  *comml);
326  MFEM_VERIFY(number_boundary_g != 0, " Please use overlapping grids.");
327 
328  Array<unsigned int> colorv;
329  colorv.SetSize(number_boundary);
330 
331  MPI_Barrier(MPI_COMM_WORLD);
332  Vector bnd(number_boundary*dim);
333  for (int i = 0; i < number_boundary; i++)
334  {
335  int idx = ess_tdof_list_int[i];
336  for (int d = 0; d < dim; d++)
337  {
338  bnd(i+d*number_boundary) = vxyz(idx + d*number_true);
339  }
340  colorv[i] = (unsigned int)color;
341  }
342  Vector interp_vals1(number_boundary);
343  finder.Interpolate(bnd, colorv, x, interp_vals1);
344 
345  // Set up the bilinear form a(.,.) on the finite element space corresponding
346  // to the Laplacian operator -Delta, by adding a Diffusion integrator.
347  ParBilinearForm *a = new ParBilinearForm(fespace);
349 
350  // Assemble the bilinear form and the corresponding linear system,
351  // applying any necessary transformations such as: eliminating boundary
352  // conditions, applying conforming constraints for non-conforming AMR,
353  // static condensation, etc.
354  a->Assemble();
355 
356  delete b;
357 
358  // Use simultaneous Schwarz iterations to iteratively solve the PDE and
359  // interpolate interdomain boundary data to impose Dirichlet boundary
360  // conditions.
361 
362  int NiterSchwarz = 100;
363  for (int schwarz = 0; schwarz < NiterSchwarz; schwarz++)
364  {
365  ParLinearForm *b = new ParLinearForm(fespace);
367  b->Assemble();
368 
369  OperatorPtr A;
370  Vector B, X;
371  a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
372 
373  // Solve the linear system A X = B.
374  // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
375  Solver *prec = NULL;
376  prec = new HypreBoomerAMG;
377  dynamic_cast<HypreBoomerAMG *>(prec)->SetPrintLevel(-1);
378  CGSolver cg(*comml);
379  cg.SetRelTol(1e-12);
380  cg.SetMaxIter(2000);
381  cg.SetPrintLevel(0);
382  cg.SetPreconditioner(*prec);
383  cg.SetOperator(*A);
384  cg.Mult(B, X);
385  delete prec;
386 
387  // Recover the solution as a finite element grid function.
388  a->RecoverFEMSolution(X, *b, x);
389 
390  // Interpolate boundary condition
391  finder.Interpolate(x, interp_vals1);
392 
393  double dxmax = std::numeric_limits<float>::min();
394  double xinf = x.Normlinf();
395  double xinfg = xinf;
396  MPI_Allreduce(&xinf, &xinfg, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
397  x.SetTrueVector();
398  Vector xt = x.GetTrueVector();
399  for (int i = 0; i < number_boundary; i++)
400  {
401  int idx = ess_tdof_list_int[i];
402  double dx = std::abs(xt(idx)-interp_vals1(i))/xinfg;
403  if (dx > dxmax) { dxmax = dx; }
404  xt(idx) = interp_vals1(i);
405  }
406  x.SetFromTrueDofs(xt);
407  double dxmaxg = dxmax;
408  MPI_Allreduce(&dxmax, &dxmaxg, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
409 
410  delete b;
411 
412  if (myid == 0)
413  {
414  std::cout << std::setprecision(8) <<
415  "Iteration: " << schwarz <<
416  ", Relative residual: " << dxmaxg << endl;
417  }
418 
419  if (dxmaxg < rel_tol) { break; }
420  }
421 
422  // Send the solution by socket to a GLVis server.
423  if (visualization)
424  {
425  char vishost[] = "localhost";
426  int visport = 19916;
427  socketstream sol_sock(vishost, visport);
428  sol_sock << "parallel " << num_procs << " " << myid << "\n";
429  sol_sock.precision(8);
430  sol_sock << "solution\n" << *pmesh << x << flush;
431  }
432 
433  // 15. Free the used memory.
434  finder.FreeData();
435  delete a;
436  delete fespace;
437  if (order > 0) { delete fec; }
438  delete pmesh;
439  delete comml;
440 
441 
442  return 0;
443 }
int Size() const
Return the logical size of the array.
Definition: array.hpp:138
Class for domain integration L(v) := (f, v)
Definition: lininteg.hpp:108
virtual void GetEssentialTrueDofs(const Array< int > &bdr_attr_is_ess, Array< int > &ess_tdof_list, int component=-1)
Definition: pfespace.cpp:1032
Conjugate gradient method.
Definition: solvers.hpp:465
A coefficient that is constant across space and time.
Definition: coefficient.hpp:84
virtual void FormLinearSystem(const Array< int > &ess_tdof_list, Vector &x, Vector &b, OperatorHandle &A, Vector &X, Vector &B, int copy_interior=0)
Form the linear system A X = B, corresponding to this bilinear form and the linear form b(...
virtual void Mult(const Vector &b, Vector &x) const
Operator application: y=A(x).
Definition: solvers.cpp:711
Pointer to an Operator of a specified type.
Definition: handle.hpp:33
HYPRE_BigInt GlobalTrueVSize() const
Definition: pfespace.hpp:285
int Size() const
Returns the size of the vector.
Definition: vector.hpp:200
void Interpolate(const Vector &point_pos, Array< unsigned int > &point_id, const GridFunction &field_in, Vector &field_out, int point_pos_ordering=Ordering::byNODES)
Definition: gslib.cpp:1263
Abstract parallel finite element space.
Definition: pfespace.hpp:28
double Normlinf() const
Returns the l_infinity norm of the vector.
Definition: vector.cpp:830
The BoomerAMG solver in hypre.
Definition: hypre.hpp:1579
Class for parallel linear form.
Definition: plinearform.hpp:26
void SetTrueVector()
Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
Definition: gridfunc.hpp:146
virtual void SetPrintLevel(int print_lvl)
Legacy method to set the level of verbosity of the solver output.
Definition: solvers.cpp:71
virtual void SetFromTrueDofs(const Vector &tv)
Set the GridFunction from the given true-dof vector.
Definition: pgridfunc.hpp:169
void Parse()
Parse the command-line options. Note that this function expects all the options provided through the ...
Definition: optparser.cpp:151
OversetFindPointsGSLIB enables use of findpts for arbitrary number of overlapping grids...
Definition: gslib.hpp:214
int Append(const T &el)
Append element &#39;el&#39; to array, resize if necessary.
Definition: array.hpp:751
constexpr char vishost[]
virtual void FreeData()
Definition: gslib.cpp:265
void Assemble()
Assembles the ParLinearForm i.e. sums over all domain/bdr integrators.
Definition: plinearform.cpp:46
double b
Definition: lissajous.cpp:42
void UniformRefinement(int i, const DSTable &, int *, int *, int *)
Definition: mesh.cpp:9816
constexpr int visport
void SetMaxIter(int max_it)
Definition: solvers.hpp:200
T Max() const
Find the maximal element in the array, using the comparison operator &lt; for class T.
Definition: array.cpp:68
void Assemble(int skip_zeros=1)
Assemble the local matrix.
int Dimension() const
Definition: mesh.hpp:1006
void PrintUsage(std::ostream &out) const
Print the usage message.
Definition: optparser.cpp:454
const Vector & GetTrueVector() const
Read only access to the (optional) internal true-dof Vector.
Definition: gridfunc.hpp:132
void AddDomainIntegrator(LinearFormIntegrator *lfi)
Adds new Domain Integrator. Assumes ownership of lfi.
Definition: linearform.cpp:41
Array< int > bdr_attributes
A list of all unique boundary attributes used by the Mesh.
Definition: mesh.hpp:270
void SetRelTol(double rtol)
Definition: solvers.hpp:198
Collection of finite elements from the same family in multiple dimensions. This class is used to matc...
Definition: fe_coll.hpp:26
void AddOption(bool *var, const char *enable_short_name, const char *enable_long_name, const char *disable_short_name, const char *disable_long_name, const char *description, bool required=false)
Add a boolean option and set &#39;var&#39; to receive the value. Enable/disable tags are used to set the bool...
Definition: optparser.hpp:82
void SetSize(int nsize)
Change the logical size of the array, keep existing entries.
Definition: array.hpp:679
HYPRE_Int HYPRE_BigInt
virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x)
double a
Definition: lissajous.cpp:41
virtual const char * Name() const
Definition: fe_coll.hpp:65
int dim
Definition: ex24.cpp:53
void SetCurvature(int order, bool discont=false, int space_dim=-1, int ordering=1) override
Definition: pmesh.cpp:2056
virtual const Array< unsigned int > & GetCode() const
Definition: gslib.hpp:191
void AddDomainIntegrator(BilinearFormIntegrator *bfi)
Adds new Domain Integrator. Assumes ownership of bfi.
void PrintOptions(std::ostream &out) const
Print the options.
Definition: optparser.cpp:324
Class for parallel bilinear form.
virtual void SetOperator(const Operator &op)
Also calls SetOperator for the preconditioner if there is one.
Definition: solvers.hpp:479
Vector data type.
Definition: vector.hpp:60
void GetNodes(Vector &node_coord) const
Definition: mesh.cpp:7908
virtual void SetPreconditioner(Solver &pr)
This should be called before SetOperator.
Definition: solvers.cpp:173
Arbitrary order H1-conforming (continuous) finite elements.
Definition: fe_coll.hpp:220
void GetInterdomainBoundaryPoints(FindPointsGSLIB &finder1, FindPointsGSLIB &finder2, Vector &mesh_nodes_1, Vector &mesh_nodes_2, Array< int > ess_tdof_list1, Array< int > ess_tdof_list2, Array< int > &ess_tdof_list1_int, Array< int > &ess_tdof_list2_int, const int dim)
void SetNodes(const Vector &node_coord)
Definition: mesh.cpp:7920
Base class for solvers.
Definition: operator.hpp:655
Class for parallel grid function.
Definition: pgridfunc.hpp:32
void FindPoints(const Vector &point_pos, Array< unsigned int > &point_id, int point_pos_ordering=Ordering::byNODES)
Definition: gslib.cpp:1187
Class for parallel meshes.
Definition: pmesh.hpp:32
void Setup(Mesh &m, const int meshid, GridFunction *gfmax=NULL, const double bb_t=0.1, const double newt_tol=1.0e-12, const int npt_max=256)
Definition: gslib.cpp:1106
int main()
bool Good() const
Return true if the command line options were parsed successfully.
Definition: optparser.hpp:150