MFEM v4.7.0
Finite element discretization library
Loading...
Searching...
No Matches
sve.hpp
Go to the documentation of this file.
1// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
2// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
3// LICENSE and NOTICE for details. LLNL-CODE-806117.
4//
5// This file is part of the MFEM library. For more information and source code
6// availability visit https://mfem.org.
7//
8// MFEM is free software; you can redistribute it and/or modify it under the
9// terms of the BSD-3 license. We welcome feedback and contributions, see file
10// CONTRIBUTING.md for details.
11
12#ifndef MFEM_SIMD_SVE_HPP
13#define MFEM_SIMD_SVE_HPP
14
15#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
16
18#include <arm_sve.h>
19
20namespace mfem
21{
22
23// Use this macro as a workaround for astyle formatting issue with 'alignas'
24#define MFEM_AUTOSIMD_ALIGN_SVE alignas(64)
25
26template <typename,int,int> struct AutoSIMD;
27
28template <> struct MFEM_AUTOSIMD_ALIGN_SVE AutoSIMD<double,8,64>
29{
30 typedef double scalar_type;
31 static constexpr int size = 8;
32 static constexpr int align_bytes = 64;
33
34 double vec[size];
35
36 AutoSIMD() = default;
37
38 AutoSIMD(const AutoSIMD &) = default;
39
40 inline MFEM_ALWAYS_INLINE double &operator[](int i)
41 {
42 return vec[i];
43 }
44
45 inline MFEM_ALWAYS_INLINE const double &operator[](int i) const
46 {
47 return vec[i];
48 }
49
50 inline MFEM_ALWAYS_INLINE AutoSIMD &operator=(const AutoSIMD &v)
51 {
52 svst1_f64(svptrue_b64(), vec, svld1_f64(svptrue_b64(),v.vec));
53 return *this;
54 }
55
56 inline MFEM_ALWAYS_INLINE AutoSIMD &operator=(const double &e)
57 {
58 svst1_f64(svptrue_b64(), vec, svdup_n_f64(e));
59 return *this;
60 }
61
62 inline MFEM_ALWAYS_INLINE AutoSIMD &operator+=(const AutoSIMD &v)
63 {
64 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
65 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
66 svst1_f64(svptrue_b64(), vec, svadd_f64_z(svptrue_b64(),vd,vvd));
67 return *this;
68 }
69
70 inline MFEM_ALWAYS_INLINE AutoSIMD &operator+=(const double &e)
71 {
72 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
73 svst1_f64(svptrue_b64(), vec, svadd_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
74 return *this;
75 }
76
77 inline MFEM_ALWAYS_INLINE AutoSIMD &operator-=(const AutoSIMD &v)
78 {
79 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
80 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
81 svst1_f64(svptrue_b64(), vec, svsub_f64_z(svptrue_b64(),vd,vvd));
82 return *this;
83 }
84
85 inline MFEM_ALWAYS_INLINE AutoSIMD &operator-=(const double &e)
86 {
87 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
88 svst1_f64(svptrue_b64(), vec, svsub_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
89 return *this;
90 }
91
92 inline MFEM_ALWAYS_INLINE AutoSIMD &operator*=(const AutoSIMD &v)
93 {
94 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
95 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
96 svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vd,vvd));
97 return *this;
98 }
99
100 inline MFEM_ALWAYS_INLINE AutoSIMD &operator*=(const double &e)
101 {
102 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
103 svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
104 return *this;
105 }
106
107 inline MFEM_ALWAYS_INLINE AutoSIMD &operator/=(const AutoSIMD &v)
108 {
109 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
110 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
111 svst1_f64(svptrue_b64(), vec, svdiv_f64_z(svptrue_b64(),vd,vvd));
112 return *this;
113 }
114
115 inline MFEM_ALWAYS_INLINE AutoSIMD &operator/=(const double &e)
116 {
117 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
118 svst1_f64(svptrue_b64(), vec, svdiv_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
119 return *this;
120 }
121
122 inline MFEM_ALWAYS_INLINE AutoSIMD operator-() const
123 {
124 AutoSIMD r;
125 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
126 svst1_f64(svptrue_b64(), r.vec, svneg_f64_z(svptrue_b64(),vd));
127 return r;
128 }
129
130 inline MFEM_ALWAYS_INLINE AutoSIMD operator+() const
131 {
132 return *this;
133 }
134
135 inline MFEM_ALWAYS_INLINE AutoSIMD operator+(const AutoSIMD &v) const
136 {
137 AutoSIMD r;
138 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
139 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
140 svst1_f64(svptrue_b64(), r.vec, svadd_f64_z(svptrue_b64(),vd,vvd));
141 return r;
142 }
143
144 inline MFEM_ALWAYS_INLINE AutoSIMD operator+(const double &e) const
145 {
146 AutoSIMD r;
147 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
148 svst1_f64(svptrue_b64(), r.vec, svadd_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
149 return r;
150 }
151
152 inline MFEM_ALWAYS_INLINE AutoSIMD operator-(const AutoSIMD &v) const
153 {
154 AutoSIMD r;
155 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
156 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
157 svst1_f64(svptrue_b64(), r.vec, svsub_f64_z(svptrue_b64(),vd,vvd));
158 return r;
159 }
160
161 inline MFEM_ALWAYS_INLINE AutoSIMD operator-(const double &e) const
162 {
163 AutoSIMD r;
164 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
165 svst1_f64(svptrue_b64(), r.vec, svsub_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
166 return r;
167 }
168
169 inline MFEM_ALWAYS_INLINE AutoSIMD operator*(const AutoSIMD &v) const
170 {
171 AutoSIMD r;
172 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
173 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
174 svst1_f64(svptrue_b64(), r.vec, svmul_f64_z(svptrue_b64(),vd,vvd));
175 return r;
176 }
177
178 inline MFEM_ALWAYS_INLINE AutoSIMD operator*(const double &e) const
179 {
180 AutoSIMD r;
181 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
182 svst1_f64(svptrue_b64(), r.vec, svmul_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
183 return r;
184 }
185
186 inline MFEM_ALWAYS_INLINE AutoSIMD operator/(const AutoSIMD &v) const
187 {
188 AutoSIMD r;
189 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
190 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
191 svst1_f64(svptrue_b64(), r.vec, svdiv_f64_z(svptrue_b64(),vd,vvd));
192 return r;
193 }
194
195 inline MFEM_ALWAYS_INLINE AutoSIMD operator/(const double &e) const
196 {
197 AutoSIMD r;
198 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
199 svst1_f64(svptrue_b64(), r.vec, svdiv_f64_z(svptrue_b64(),vd,svdup_n_f64(e)));
200 return r;
201 }
202
203 inline MFEM_ALWAYS_INLINE AutoSIMD &fma(const AutoSIMD &v, const AutoSIMD &w)
204 {
205 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
206 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
207 const svfloat64_t wvd = svld1_f64(svptrue_b64(), w.vec);
208 svst1_f64(svptrue_b64(), vec, svmad_f64_z(svptrue_b64(),wvd,vd,vvd));
209 return *this;
210 }
211
212 inline MFEM_ALWAYS_INLINE AutoSIMD &fma(const AutoSIMD &v, const double &e)
213 {
214 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
215 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
216 svst1_f64(svptrue_b64(), vec, svmad_f64_z(svptrue_b64(),vvd,svdup_n_f64(e),vd));
217 return *this;
218 }
219
220 inline MFEM_ALWAYS_INLINE AutoSIMD &fma(const double &e, const AutoSIMD &v)
221 {
222 const svfloat64_t vd = svld1_f64(svptrue_b64(), vec);
223 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
224 svst1_f64(svptrue_b64(), vec, svmad_f64_z(svptrue_b64(),svdup_n_f64(e),vvd,vd));
225 return *this;
226 }
227
228 inline MFEM_ALWAYS_INLINE AutoSIMD &mul(const AutoSIMD &v, const AutoSIMD &w)
229 {
230 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
231 const svfloat64_t wvd = svld1_f64(svptrue_b64(), w.vec);
232 svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vvd,wvd));
233 return *this;
234 }
235
236 inline MFEM_ALWAYS_INLINE AutoSIMD &mul(const AutoSIMD &v,const double &e)
237 {
238 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
239 svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),vvd,svdup_n_f64(e)));
240 return *this;
241 }
242
243 inline MFEM_ALWAYS_INLINE AutoSIMD &mul(const double &e, const AutoSIMD &v)
244 {
245 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
246 svst1_f64(svptrue_b64(), vec, svmul_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
247 return *this;
248 }
249};
250
251inline MFEM_ALWAYS_INLINE
252AutoSIMD<double,8,64> operator+(const double &e, const AutoSIMD<double,8,64> &v)
253{
254 AutoSIMD<double,8,64> r;
255 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
256 svst1_f64(svptrue_b64(), r.vec, svadd_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
257 return r;
258}
259
260inline MFEM_ALWAYS_INLINE
261AutoSIMD<double,8,64> operator-(const double &e, const AutoSIMD<double,8,64> &v)
262{
263 AutoSIMD<double,8,64> r;
264 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
265 svst1_f64(svptrue_b64(), r.vec, svsub_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
266 return r;
267}
268
269inline MFEM_ALWAYS_INLINE
270AutoSIMD<double,8,64> operator*(const double &e, const AutoSIMD<double,8,64> &v)
271{
272 AutoSIMD<double,8,64> r;
273 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
274 svst1_f64(svptrue_b64(), r.vec, svmul_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
275 return r;
276}
277
278inline MFEM_ALWAYS_INLINE
279AutoSIMD<double,8,64> operator/(const double &e, const AutoSIMD<double,8,64> &v)
280{
281 AutoSIMD<double,8,64> r;
282 const svfloat64_t vvd = svld1_f64(svptrue_b64(), v.vec);
283 svst1_f64(svptrue_b64(), r.vec, svdiv_f64_z(svptrue_b64(),svdup_n_f64(e),vvd));
284 return r;
285}
286
287} // namespace mfem
288
289#endif // __aarch64__ && __ARM_FEATURE_SVE
290
291#endif // MFEM_SIMD_SVE_HPP
MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
Return a suitable MemoryClass from a pair of MemoryClasses.
MFEM_ALWAYS_INLINE AutoSIMD< scalar_t, S, A > operator+(const scalar_t &e, const AutoSIMD< scalar_t, S, A > &v)
Definition auto.hpp:238
MFEM_ALWAYS_INLINE AutoSIMD< scalar_t, S, A > operator-(const scalar_t &e, const AutoSIMD< scalar_t, S, A > &v)
Definition auto.hpp:249
MFEM_ALWAYS_INLINE AutoSIMD< scalar_t, S, A > operator/(const scalar_t &e, const AutoSIMD< scalar_t, S, A > &v)
Definition auto.hpp:271
MFEM_ALWAYS_INLINE AutoSIMD & operator*=(const double &e)
Definition sve.hpp:100
AutoSIMD(const AutoSIMD &)=default
MFEM_ALWAYS_INLINE const double & operator[](int i) const
Definition sve.hpp:45
MFEM_ALWAYS_INLINE AutoSIMD operator+() const
Definition sve.hpp:130
MFEM_ALWAYS_INLINE AutoSIMD & fma(const double &e, const AutoSIMD &v)
Definition sve.hpp:220
MFEM_ALWAYS_INLINE AutoSIMD operator*(const AutoSIMD &v) const
Definition sve.hpp:169
MFEM_ALWAYS_INLINE AutoSIMD & operator=(const AutoSIMD &v)
Definition sve.hpp:50
MFEM_ALWAYS_INLINE AutoSIMD & operator-=(const AutoSIMD &v)
Definition sve.hpp:77
MFEM_ALWAYS_INLINE AutoSIMD & fma(const AutoSIMD &v, const AutoSIMD &w)
Definition sve.hpp:203
MFEM_ALWAYS_INLINE AutoSIMD & operator-=(const double &e)
Definition sve.hpp:85
MFEM_ALWAYS_INLINE AutoSIMD & operator+=(const double &e)
Definition sve.hpp:70
MFEM_ALWAYS_INLINE AutoSIMD & operator+=(const AutoSIMD &v)
Definition sve.hpp:62
MFEM_ALWAYS_INLINE AutoSIMD operator+(const AutoSIMD &v) const
Definition sve.hpp:135
MFEM_ALWAYS_INLINE AutoSIMD operator/(const double &e) const
Definition sve.hpp:195
MFEM_ALWAYS_INLINE AutoSIMD & fma(const AutoSIMD &v, const double &e)
Definition sve.hpp:212
MFEM_ALWAYS_INLINE AutoSIMD & operator=(const double &e)
Definition sve.hpp:56
MFEM_ALWAYS_INLINE double & operator[](int i)
Definition sve.hpp:40
MFEM_ALWAYS_INLINE AutoSIMD operator+(const double &e) const
Definition sve.hpp:144
MFEM_ALWAYS_INLINE AutoSIMD operator-() const
Definition sve.hpp:122
MFEM_ALWAYS_INLINE AutoSIMD operator*(const double &e) const
Definition sve.hpp:178
MFEM_ALWAYS_INLINE AutoSIMD & operator*=(const AutoSIMD &v)
Definition sve.hpp:92
MFEM_ALWAYS_INLINE AutoSIMD & operator/=(const double &e)
Definition sve.hpp:115
MFEM_ALWAYS_INLINE AutoSIMD operator-(const AutoSIMD &v) const
Definition sve.hpp:152
MFEM_ALWAYS_INLINE AutoSIMD operator-(const double &e) const
Definition sve.hpp:161
MFEM_ALWAYS_INLINE AutoSIMD & operator/=(const AutoSIMD &v)
Definition sve.hpp:107
MFEM_ALWAYS_INLINE AutoSIMD & mul(const AutoSIMD &v, const double &e)
Definition sve.hpp:236
MFEM_ALWAYS_INLINE AutoSIMD & mul(const double &e, const AutoSIMD &v)
Definition sve.hpp:243
MFEM_ALWAYS_INLINE AutoSIMD & mul(const AutoSIMD &v, const AutoSIMD &w)
Definition sve.hpp:228
MFEM_ALWAYS_INLINE AutoSIMD operator/(const AutoSIMD &v) const
Definition sve.hpp:186
scalar_t vec[size]
Definition auto.hpp:30