MFEM  v4.5.2
Finite element discretization library
sve.hpp
Go to the documentation of this file.
1 // Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
2 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
3 // LICENSE and NOTICE for details. LLNL-CODE-806117.
4 //
5 // This file is part of the MFEM library. For more information and source code
6 // availability visit https://mfem.org.
7 //
8 // MFEM is free software; you can redistribute it and/or modify it under the
9 // terms of the BSD-3 license. We welcome feedback and contributions, see file
10 // CONTRIBUTING.md for details.
11 
12 #ifndef MFEM_SIMD_SVE_HPP
13 #define MFEM_SIMD_SVE_HPP
14 
15 #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
16 
17 #include "../../config/tconfig.hpp"
18 #include <arm_sve.h>
19 
20 namespace mfem
21 {
22 
23 // Use this macro as a workaround for astyle formatting issue with 'alignas'
24 #define MFEM_AUTOSIMD_ALIGN_SVE alignas(64)
25 
26 template <typename,int,int> struct AutoSIMD;
27 
28 template <> struct MFEM_AUTOSIMD_ALIGN_SVE AutoSIMD<double,8,64>
29 {
30  typedef double scalar_type;
31  static constexpr int size = 8;
32  static constexpr int align_bytes = 64;
33 
34  double vec[size];
35 
36  AutoSIMD() = default;
37 
38  AutoSIMD(const AutoSIMD &) = default;
39 
40  inline MFEM_ALWAYS_INLINE double &operator[](int i)
41  {
42  return vec[i];
43  }
44 
45  inline MFEM_ALWAYS_INLINE const double &operator[](int i) const
46  {
47  return vec[i];
48  }
49 
50  inline MFEM_ALWAYS_INLINE AutoSIMD &operator=(const AutoSIMD &v)
51  {
52  svst1_f64(svptrue_b64(), vec, svld1_f64(svptrue_b64(),v.vec));
53  return *this;
54  }
55 
56  inline MFEM_ALWAYS_INLINE AutoSIMD &operator=(const double &e)
57  {
58  svst1_f64(svptrue_b64(), vec, svdup_n_f64(e));
59  return *this;
60  }
61 
62  inline MFEM_ALWAYS_INLINE AutoSIMD &operator+=(const AutoSIMD &v)
63  {
64  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
65  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
66  svst1_f64(svptrue_b64(), vec, svadd_f64_z(svptrue_b64(),vd,vvd));
67  return *this;
68  }
69 
70  inline MFEM_ALWAYS_INLINE AutoSIMD &operator+=(const double &e)
71  {
72  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
73  svst1_f64(svptrue_b64(), vec, svadd_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
74  return *this;
75  }
76 
77  inline MFEM_ALWAYS_INLINE AutoSIMD &operator-=(const AutoSIMD &v)
78  {
79  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
80  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
81  svst1_f64(svptrue_b64(), vec, svsub_f64_z(svptrue_b64(),vd,vvd));
82  return *this;
83  }
84 
85  inline MFEM_ALWAYS_INLINE AutoSIMD &operator-=(const double &e)
86  {
87  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
88  svst1_f64(svptrue_b64(), vec, svsub_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
89  return *this;
90  }
91 
92  inline MFEM_ALWAYS_INLINE AutoSIMD &operator*=(const AutoSIMD &v)
93  {
94  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
95  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
96  svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vd,vvd));
97  return *this;
98  }
99 
100  inline MFEM_ALWAYS_INLINE AutoSIMD &operator*=(const double &e)
101  {
102  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
103  svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
104  return *this;
105  }
106 
107  inline MFEM_ALWAYS_INLINE AutoSIMD &operator/=(const AutoSIMD &v)
108  {
109  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
110  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
111  svst1_f64(svptrue_b64(), vec, svdiv_f64_z(svptrue_b64(),vd,vvd));
112  return *this;
113  }
114 
115  inline MFEM_ALWAYS_INLINE AutoSIMD &operator/=(const double &e)
116  {
117  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
118  svst1_f64(svptrue_b64(), vec, svdiv_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
119  return *this;
120  }
121 
122  inline MFEM_ALWAYS_INLINE AutoSIMD operator-() const
123  {
124  AutoSIMD r;
125  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
126  svst1_f64(svptrue_b64(), r.vec, svneg_f64_z(svptrue_b64(),vd));
127  return r;
128  }
129 
130  inline MFEM_ALWAYS_INLINE AutoSIMD operator+() const
131  {
132  return *this;
133  }
134 
135  inline MFEM_ALWAYS_INLINE AutoSIMD operator+(const AutoSIMD &v) const
136  {
137  AutoSIMD r;
138  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
139  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
140  svst1_f64(svptrue_b64(), r.vec, svadd_f64_z(svptrue_b64(),vd,vvd));
141  return r;
142  }
143 
144  inline MFEM_ALWAYS_INLINE AutoSIMD operator+(const double &e) const
145  {
146  AutoSIMD r;
147  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
148  svst1_f64(svptrue_b64(), r.vec, svadd_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
149  return r;
150  }
151 
152  inline MFEM_ALWAYS_INLINE AutoSIMD operator-(const AutoSIMD &v) const
153  {
154  AutoSIMD r;
155  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
156  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
157  svst1_f64(svptrue_b64(), r.vec, svsub_f64_z(svptrue_b64(),vd,vvd));
158  return r;
159  }
160 
161  inline MFEM_ALWAYS_INLINE AutoSIMD operator-(const double &e) const
162  {
163  AutoSIMD r;
164  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
165  svst1_f64(svptrue_b64(), r.vec, svsub_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
166  return r;
167  }
168 
169  inline MFEM_ALWAYS_INLINE AutoSIMD operator*(const AutoSIMD &v) const
170  {
171  AutoSIMD r;
172  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
173  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
174  svst1_f64(svptrue_b64(), r.vec, svmul_f64_z(svptrue_b64(),vd,vvd));
175  return r;
176  }
177 
178  inline MFEM_ALWAYS_INLINE AutoSIMD operator*(const double &e) const
179  {
180  AutoSIMD r;
181  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
182  svst1_f64(svptrue_b64(), r.vec, svmul_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
183  return r;
184  }
185 
186  inline MFEM_ALWAYS_INLINE AutoSIMD operator/(const AutoSIMD &v) const
187  {
188  AutoSIMD r;
189  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
190  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
191  svst1_f64(svptrue_b64(), r.vec, svdiv_f64_z(svptrue_b64(),vd,vvd));
192  return r;
193  }
194 
195  inline MFEM_ALWAYS_INLINE AutoSIMD operator/(const double &e) const
196  {
197  AutoSIMD r;
198  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
199  svst1_f64(svptrue_b64(), r.vec, svdiv_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
200  return r;
201  }
202 
203  inline MFEM_ALWAYS_INLINE AutoSIMD &fma(const AutoSIMD &v, const AutoSIMD &w)
204  {
205  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
206  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
207  const svfloat64_t wvd = svld1_f64(svptrue_b64(), w.vec);
208  svst1_f64(svptrue_b64(), vec, svmad_f64_z(svptrue_b64(),wvd,vd,vvd));
209  return *this;
210  }
211 
212  inline MFEM_ALWAYS_INLINE AutoSIMD &fma(const AutoSIMD &v, const double &e)
213  {
214  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
215  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
216  svst1_f64(svptrue_b64(), vec, svmad_f64_z(svptrue_b64(),vvd,svdup_n_f64(e),vd));
217  return *this;
218  }
219 
220  inline MFEM_ALWAYS_INLINE AutoSIMD &fma(const double &e, const AutoSIMD &v)
221  {
222  const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
223  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
224  svst1_f64(svptrue_b64(), vec, svmad_f64_z(svptrue_b64(),svdup_n_f64(e),vvd,vd));
225  return *this;
226  }
227 
228  inline MFEM_ALWAYS_INLINE AutoSIMD &mul(const AutoSIMD &v, const AutoSIMD &w)
229  {
230  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
231  const svfloat64_t wvd = svld1_f64(svptrue_b64(), w.vec);
232  svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vvd,wvd));
233  return *this;
234  }
235 
236  inline MFEM_ALWAYS_INLINE AutoSIMD &mul(const AutoSIMD &v,const double &e)
237  {
238  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
239  svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vvd,svdup_n_f64(e)));
240  return *this;
241  }
242 
243  inline MFEM_ALWAYS_INLINE AutoSIMD &mul(const double &e, const AutoSIMD &v)
244  {
245  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
246  svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
247  return *this;
248  }
249 };
250 
251 inline MFEM_ALWAYS_INLINE
252 AutoSIMD<double,8,64> operator+(const double &e, const AutoSIMD<double,8,64> &v)
253 {
254  AutoSIMD<double,8,64> r;
255  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
256  svst1_f64(svptrue_b64(), r.vec, svadd_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
257  return r;
258 }
259 
260 inline MFEM_ALWAYS_INLINE
261 AutoSIMD<double,8,64> operator-(const double &e, const AutoSIMD<double,8,64> &v)
262 {
263  AutoSIMD<double,8,64> r;
264  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
265  svst1_f64(svptrue_b64(), r.vec, svsub_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
266  return r;
267 }
268 
269 inline MFEM_ALWAYS_INLINE
270 AutoSIMD<double,8,64> operator*(const double &e, const AutoSIMD<double,8,64> &v)
271 {
272  AutoSIMD<double,8,64> r;
273  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
274  svst1_f64(svptrue_b64(), r.vec, svmul_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
275  return r;
276 }
277 
278 inline MFEM_ALWAYS_INLINE
279 AutoSIMD<double,8,64> operator/(const double &e, const AutoSIMD<double,8,64> &v)
280 {
281  AutoSIMD<double,8,64> r;
282  const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
283  svst1_f64(svptrue_b64(), r.vec, svdiv_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
284  return r;
285 }
286 
287 } // namespace mfem
288 
289 #endif // __aarch64__ && __ARM_FEATURE_SVE
290 
291 #endif // MFEM_SIMD_SVE_HPP
MFEM_ALWAYS_INLINE AutoSIMD & fma(const double &e, const AutoSIMD &v)
Definition: sve.hpp:220
MFEM_ALWAYS_INLINE AutoSIMD operator+() const
Definition: sve.hpp:130
MFEM_ALWAYS_INLINE AutoSIMD operator*(const double &e) const
Definition: sve.hpp:178
MFEM_ALWAYS_INLINE AutoSIMD & operator=(const double &e)
Definition: sve.hpp:56
MFEM_ALWAYS_INLINE AutoSIMD & fma(const AutoSIMD &v, const double &e)
Definition: sve.hpp:212
MFEM_ALWAYS_INLINE AutoSIMD< scalar_t, S, A > operator/(const scalar_t &e, const AutoSIMD< scalar_t, S, A > &v)
Definition: auto.hpp:271
MFEM_ALWAYS_INLINE AutoSIMD operator-(const AutoSIMD &v) const
Definition: sve.hpp:152
MFEM_ALWAYS_INLINE AutoSIMD & mul(const AutoSIMD &v, const AutoSIMD &w)
Definition: sve.hpp:228
MFEM_ALWAYS_INLINE AutoSIMD & operator/=(const double &e)
Definition: sve.hpp:115
MFEM_ALWAYS_INLINE AutoSIMD & operator+=(const AutoSIMD &v)
Definition: sve.hpp:62
MFEM_ALWAYS_INLINE AutoSIMD & fma(const AutoSIMD &v, const AutoSIMD &w)
Definition: sve.hpp:203
scalar_t vec[size]
Definition: auto.hpp:30
MFEM_ALWAYS_INLINE AutoSIMD< scalar_t, S, A > operator+(const scalar_t &e, const AutoSIMD< scalar_t, S, A > &v)
Definition: auto.hpp:238
MFEM_ALWAYS_INLINE const double & operator[](int i) const
Definition: sve.hpp:45
MFEM_ALWAYS_INLINE AutoSIMD operator+(const AutoSIMD &v) const
Definition: sve.hpp:135
MFEM_ALWAYS_INLINE AutoSIMD & mul(const AutoSIMD &v, const double &e)
Definition: sve.hpp:236
MFEM_ALWAYS_INLINE AutoSIMD & operator+=(const double &e)
Definition: sve.hpp:70
MFEM_ALWAYS_INLINE AutoSIMD operator+(const double &e) const
Definition: sve.hpp:144
MFEM_ALWAYS_INLINE double & operator[](int i)
Definition: sve.hpp:40
MFEM_ALWAYS_INLINE AutoSIMD & operator-=(const AutoSIMD &v)
Definition: sve.hpp:77
MFEM_ALWAYS_INLINE AutoSIMD operator-(const double &e) const
Definition: sve.hpp:161
MFEM_ALWAYS_INLINE AutoSIMD & operator/=(const AutoSIMD &v)
Definition: sve.hpp:107
MFEM_ALWAYS_INLINE AutoSIMD & operator=(const AutoSIMD &v)
Definition: sve.hpp:50
MFEM_ALWAYS_INLINE AutoSIMD operator-() const
Definition: sve.hpp:122
MFEM_ALWAYS_INLINE AutoSIMD operator/(const AutoSIMD &v) const
Definition: sve.hpp:186
MFEM_ALWAYS_INLINE AutoSIMD & operator*=(const AutoSIMD &v)
Definition: sve.hpp:92
MFEM_ALWAYS_INLINE AutoSIMD & mul(const double &e, const AutoSIMD &v)
Definition: sve.hpp:243
MFEM_ALWAYS_INLINE AutoSIMD operator*(const AutoSIMD &v) const
Definition: sve.hpp:169
MFEM_ALWAYS_INLINE AutoSIMD & operator*=(const double &e)
Definition: sve.hpp:100
MFEM_ALWAYS_INLINE AutoSIMD operator/(const double &e) const
Definition: sve.hpp:195
MFEM_ALWAYS_INLINE AutoSIMD & operator-=(const double &e)
Definition: sve.hpp:85
MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
Return a suitable MemoryClass from a pair of MemoryClasses.
MFEM_ALWAYS_INLINE AutoSIMD< scalar_t, S, A > operator-(const scalar_t &e, const AutoSIMD< scalar_t, S, A > &v)
Definition: auto.hpp:249