$darkmode
Eigen  5.0.1-dev
TypeCasting.h
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@google.com>
5 // Copyright (C) 2020 Antonio Sanchez <cantonios@google.com>
6 //
7 // This Source Code Form is subject to the terms of the Mozilla
8 // Public License v. 2.0. If a copy of the MPL was not distributed
9 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10 
11 #ifndef EIGEN_TYPE_CASTING_NEON_H
12 #define EIGEN_TYPE_CASTING_NEON_H
13 
14 // IWYU pragma: private
15 #include "../../InternalHeaderCheck.h"
16 
17 namespace Eigen {
18 
19 namespace internal {
20 
21 //==============================================================================
22 // preinterpret (truncation operations)
23 //==============================================================================
24 
25 template <>
26 EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet16c>(const Packet16c& a) {
27  return Packet8c(vget_low_s8(a));
28 }
29 template <>
30 EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet8c>(const Packet8c& a) {
31  return Packet4c(vget_lane_s32(vreinterpret_s32_s8(a), 0));
32 }
33 template <>
34 EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet16c>(const Packet16c& a) {
35  return preinterpret<Packet4c>(preinterpret<Packet8c>(a));
36 }
37 
38 template <>
39 EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet16uc>(const Packet16uc& a) {
40  return Packet8uc(vget_low_u8(a));
41 }
42 template <>
43 EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet8uc>(const Packet8uc& a) {
44  return Packet4uc(vget_lane_u32(vreinterpret_u32_u8(a), 0));
45 }
46 template <>
47 EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet16uc>(const Packet16uc& a) {
48  return preinterpret<Packet4uc>(preinterpret<Packet8uc>(a));
49 }
50 
51 template <>
52 EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet8s>(const Packet8s& a) {
53  return Packet4s(vget_low_s16(a));
54 }
55 
56 template <>
57 EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet8us>(const Packet8us& a) {
58  return Packet4us(vget_low_u16(a));
59 }
60 
61 template <>
62 EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet4i>(const Packet4i& a) {
63  return Packet2i(vget_low_s32(a));
64 }
65 template <>
66 EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet4ui>(const Packet4ui& a) {
67  return Packet2ui(vget_low_u32(a));
68 }
69 
70 template <>
71 EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet4f>(const Packet4f& a) {
72  return Packet2f(vget_low_f32(a));
73 }
74 
75 //==============================================================================
76 // preinterpret
77 //==============================================================================
78 template <>
79 EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
80  return Packet2f(vreinterpret_f32_s32(a));
81 }
82 template <>
83 EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
84  return Packet2f(vreinterpret_f32_u32(a));
85 }
86 template <>
87 EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
88  return Packet4f(vreinterpretq_f32_s32(a));
89 }
90 template <>
91 EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
92  return Packet4f(vreinterpretq_f32_u32(a));
93 }
94 
95 template <>
96 EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
97  return static_cast<Packet4c>(a);
98 }
99 template <>
100 EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
101  return Packet8c(vreinterpret_s8_u8(a));
102 }
103 template <>
104 EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
105  return Packet16c(vreinterpretq_s8_u8(a));
106 }
107 
108 template <>
109 EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
110  return static_cast<Packet4uc>(a);
111 }
112 template <>
113 EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
114  return Packet8uc(vreinterpret_u8_s8(a));
115 }
116 template <>
117 EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
118  return Packet16uc(vreinterpretq_u8_s8(a));
119 }
120 
121 template <>
122 EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
123  return Packet4s(vreinterpret_s16_u16(a));
124 }
125 template <>
126 EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
127  return Packet8s(vreinterpretq_s16_u16(a));
128 }
129 template <>
130 EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
131  return Packet4us(vreinterpret_u16_s16(a));
132 }
133 template <>
134 EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
135  return Packet8us(vreinterpretq_u16_s16(a));
136 }
137 
138 template <>
139 EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
140  return Packet2i(vreinterpret_s32_f32(a));
141 }
142 template <>
143 EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
144  return Packet2i(vreinterpret_s32_u32(a));
145 }
146 template <>
147 EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
148  return Packet4i(vreinterpretq_s32_f32(a));
149 }
150 template <>
151 EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
152  return Packet4i(vreinterpretq_s32_u32(a));
153 }
154 
155 template <>
156 EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
157  return Packet2ui(vreinterpret_u32_f32(a));
158 }
159 template <>
160 EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
161  return Packet2ui(vreinterpret_u32_s32(a));
162 }
163 template <>
164 EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
165  return Packet4ui(vreinterpretq_u32_f32(a));
166 }
167 template <>
168 EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
169  return Packet4ui(vreinterpretq_u32_s32(a));
170 }
171 
172 template <>
173 EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
174  return Packet2l(vreinterpretq_s64_u64(a));
175 }
176 template <>
177 EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
178  return Packet2ul(vreinterpretq_u64_s64(a));
179 }
180 
181 //==============================================================================
182 // pcast, SrcType = float
183 //==============================================================================
184 
185 template <>
186 struct type_casting_traits<float, numext::int64_t> {
187  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
188 };
189 template <>
190 struct type_casting_traits<float, numext::uint64_t> {
191  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
192 };
193 // If float64 exists, first convert to that to keep as much precision as possible.
194 #if EIGEN_ARCH_ARM64
195 template <>
196 EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
197  // Discard second half of input.
198  return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
199 }
200 template <>
201 EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
202  return vcvtq_s64_f64(vcvt_f64_f32(a));
203 }
204 template <>
205 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
206  // Discard second half of input.
207  return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
208 }
209 template <>
210 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
211  return vcvtq_u64_f64(vcvt_f64_f32(a));
212 }
213 #else
214 template <>
215 EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
216  // Discard second half of input.
217  return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a)));
218 }
219 template <>
220 EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
221  return vmovl_s32(vcvt_s32_f32(a));
222 }
223 template <>
224 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
225  // Discard second half of input.
226  return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a)));
227 }
228 template <>
229 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
230  // Discard second half of input.
231  return vmovl_u32(vcvt_u32_f32(a));
232 }
233 #endif // EIGEN_ARCH_ARM64
234 
235 template <>
236 struct type_casting_traits<float, numext::int32_t> {
237  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
238 };
239 template <>
240 EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
241  return vcvtq_s32_f32(a);
242 }
243 template <>
244 EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
245  return vcvt_s32_f32(a);
246 }
247 
248 template <>
249 struct type_casting_traits<float, numext::uint32_t> {
250  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
251 };
252 template <>
253 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
254  return vcvtq_u32_f32(a);
255 }
256 template <>
257 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
258  return vcvt_u32_f32(a);
259 }
260 
261 template <>
262 struct type_casting_traits<float, numext::int16_t> {
263  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
264 };
265 template <>
266 EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
267  return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
268 }
269 template <>
270 EIGEN_STRONG_INLINE Packet4s pcast<Packet4f, Packet4s>(const Packet4f& a) {
271  return vmovn_s32(vcvtq_s32_f32(a));
272 }
273 template <>
274 EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
275  return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
276 }
277 
278 template <>
279 struct type_casting_traits<float, numext::uint16_t> {
280  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
281 };
282 template <>
283 EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
284  return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
285 }
286 template <>
287 EIGEN_STRONG_INLINE Packet4us pcast<Packet4f, Packet4us>(const Packet4f& a) {
288  return vmovn_u32(vcvtq_u32_f32(a));
289 }
290 template <>
291 EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
292  return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
293 }
294 
295 template <>
296 struct type_casting_traits<float, numext::int8_t> {
297  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
298 };
299 template <>
300 EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
301  const Packet4f& d) {
302  const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
303  const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
304  return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
305 }
306 template <>
307 EIGEN_STRONG_INLINE Packet8c pcast<Packet4f, Packet8c>(const Packet4f& a, const Packet4f& b) {
308  const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
309  return vmovn_s16(ab_s16);
310 }
311 template <>
312 EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
313  const Packet2f& d) {
314  const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
315  const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
316  return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
317 }
318 template <>
319 EIGEN_STRONG_INLINE Packet4c pcast<Packet4f, Packet4c>(const Packet4f& a) {
320  const int32x4_t a_s32x4 = vcvtq_s32_f32(a);
321  const int16x4_t a_s16x4 = vmovn_s32(a_s32x4);
322  const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
323  const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
324  return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
325 }
326 
327 template <>
328 struct type_casting_traits<float, numext::uint8_t> {
329  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
330 };
331 template <>
332 EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
333  const Packet4f& d) {
334  return preinterpret<Packet16uc>(pcast<Packet4f, Packet16c>(a, b, c, d));
335 }
336 template <>
337 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4f, Packet8uc>(const Packet4f& a, const Packet4f& b) {
338  return preinterpret<Packet8uc>(pcast<Packet4f, Packet8c>(a, b));
339 }
340 template <>
341 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
342  const Packet2f& d) {
343  return preinterpret<Packet8uc>(pcast<Packet2f, Packet8c>(a, b, c, d));
344 }
345 template <>
346 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4f, Packet4uc>(const Packet4f& a) {
347  return static_cast<Packet4uc>(pcast<Packet4f, Packet4c>(a));
348 }
349 
350 //==============================================================================
351 // pcast, SrcType = int8_t
352 //==============================================================================
353 template <>
354 struct type_casting_traits<numext::int8_t, float> {
355  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
356 };
357 template <>
358 EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
359  // Discard all but first 4 bytes.
360  return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
361 }
362 template <>
363 EIGEN_STRONG_INLINE Packet4f pcast<Packet4c, Packet4f>(const Packet4c& a) {
364  return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(a))))));
365 }
366 template <>
367 EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
368  // Discard all but first 2 bytes.
369  return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
370 }
371 
372 template <>
373 struct type_casting_traits<numext::int8_t, numext::int64_t> {
374  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
375 };
376 template <>
377 EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
378  // Discard all but first two bytes.
379  return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
380 }
381 
382 template <>
383 struct type_casting_traits<numext::int8_t, numext::uint64_t> {
384  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
385 };
386 template <>
387 EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
388  return preinterpret<Packet2ul>(pcast<Packet16c, Packet2l>(a));
389 }
390 
391 template <>
392 struct type_casting_traits<numext::int8_t, numext::int32_t> {
393  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
394 };
395 template <>
396 EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
397  // Discard all but first 4 bytes.
398  return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
399 }
400 template <>
401 EIGEN_STRONG_INLINE Packet4i pcast<Packet8c, Packet4i>(const Packet8c& a) {
402  return vmovl_s16(vget_low_s16(vmovl_s8(a)));
403 }
404 template <>
405 EIGEN_STRONG_INLINE Packet4i pcast<Packet4c, Packet4i>(const Packet4c& a) {
406  return pcast<Packet8c, Packet4i>(vreinterpret_s8_s32(vdup_n_s32(a)));
407 }
408 template <>
409 EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
410  // Discard all but first 2 bytes.
411  return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
412 }
413 
414 template <>
415 struct type_casting_traits<numext::int8_t, numext::uint32_t> {
416  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
417 };
418 template <>
419 EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
420  return preinterpret<Packet4ui>(pcast<Packet16c, Packet4i>(a));
421 }
422 template <>
423 EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
424  return preinterpret<Packet2ui>(pcast<Packet8c, Packet2i>(a));
425 }
426 template <>
427 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4c, Packet4ui>(const Packet4c& a) {
428  return preinterpret<Packet4ui>(pcast<Packet4c, Packet4i>(a));
429 }
430 
431 template <>
432 struct type_casting_traits<numext::int8_t, numext::int16_t> {
433  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
434 };
435 template <>
436 EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
437  // Discard second half of input.
438  return vmovl_s8(vget_low_s8(a));
439 }
440 template <>
441 EIGEN_STRONG_INLINE Packet8s pcast<Packet8c, Packet8s>(const Packet8c& a) {
442  return vmovl_s8(a);
443 }
444 template <>
445 EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
446  // Discard second half of input.
447  return vget_low_s16(vmovl_s8(a));
448 }
449 template <>
450 EIGEN_STRONG_INLINE Packet4s pcast<Packet4c, Packet4s>(const Packet4c& a) {
451  return pcast<Packet8c, Packet4s>(vreinterpret_s8_s32(vdup_n_s32(a)));
452 }
453 
454 template <>
455 struct type_casting_traits<numext::int8_t, numext::uint16_t> {
456  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
457 };
458 template <>
459 EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
460  return preinterpret<Packet8us>(pcast<Packet16c, Packet8s>(a));
461 }
462 template <>
463 EIGEN_STRONG_INLINE Packet8us pcast<Packet8c, Packet8us>(const Packet8c& a) {
464  return preinterpret<Packet8us>(pcast<Packet8c, Packet8s>(a));
465 }
466 template <>
467 EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
468  return preinterpret<Packet4us>(pcast<Packet8c, Packet4s>(a));
469 }
470 template <>
471 EIGEN_STRONG_INLINE Packet4us pcast<Packet4c, Packet4us>(const Packet4c& a) {
472  return preinterpret<Packet4us>(pcast<Packet4c, Packet4s>(a));
473 }
474 
475 //==============================================================================
476 // pcast, SrcType = uint8_t
477 //==============================================================================
478 template <>
479 struct type_casting_traits<numext::uint8_t, float> {
480  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
481 };
482 template <>
483 EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
484  // Discard all but first 4 bytes.
485  return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
486 }
487 template <>
488 EIGEN_STRONG_INLINE Packet4f pcast<Packet4uc, Packet4f>(const Packet4uc& a) {
489  return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))))));
490 }
491 template <>
492 EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
493  // Discard all but first 2 bytes.
494  return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
495 }
496 
497 template <>
498 struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
499  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
500 };
501 template <>
502 EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
503  // Discard all but first two bytes.
504  return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
505 }
506 
507 template <>
508 struct type_casting_traits<numext::uint8_t, numext::int64_t> {
509  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
510 };
511 template <>
512 EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
513  return preinterpret<Packet2l>(pcast<Packet16uc, Packet2ul>(a));
514 }
515 
516 template <>
517 struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
518  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
519 };
520 template <>
521 EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
522  // Discard all but first 4 bytes.
523  return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
524 }
525 template <>
526 EIGEN_STRONG_INLINE Packet4ui pcast<Packet8uc, Packet4ui>(const Packet8uc& a) {
527  return vmovl_u16(vget_low_u16(vmovl_u8(a)));
528 }
529 template <>
530 EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
531  // Discard all but first 2 bytes.
532  return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
533 }
534 template <>
535 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4uc, Packet4ui>(const Packet4uc& a) {
536  return pcast<Packet8uc, Packet4ui>(vreinterpret_u8_u32(vdup_n_u32(a)));
537 }
538 
539 template <>
540 struct type_casting_traits<numext::uint8_t, numext::int32_t> {
541  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
542 };
543 template <>
544 EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
545  return preinterpret<Packet4i>(pcast<Packet16uc, Packet4ui>(a));
546 }
547 template <>
548 EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
549  return preinterpret<Packet2i>(pcast<Packet8uc, Packet2ui>(a));
550 }
551 template <>
552 EIGEN_STRONG_INLINE Packet4i pcast<Packet4uc, Packet4i>(const Packet4uc& a) {
553  return preinterpret<Packet4i>(pcast<Packet4uc, Packet4ui>(a));
554 }
555 
556 template <>
557 struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
558  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
559 };
560 template <>
561 EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
562  // Discard second half of input.
563  return vmovl_u8(vget_low_u8(a));
564 }
565 template <>
566 EIGEN_STRONG_INLINE Packet8us pcast<Packet8uc, Packet8us>(const Packet8uc& a) {
567  return vmovl_u8(a);
568 }
569 template <>
570 EIGEN_STRONG_INLINE Packet4us pcast<Packet4uc, Packet4us>(const Packet4uc& a) {
571  return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))));
572 }
573 
574 template <>
575 struct type_casting_traits<numext::uint8_t, numext::int16_t> {
576  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
577 };
578 template <>
579 EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
580  return preinterpret<Packet8s>(pcast<Packet16uc, Packet8us>(a));
581 }
582 template <>
583 EIGEN_STRONG_INLINE Packet8s pcast<Packet8uc, Packet8s>(const Packet8uc& a) {
584  return preinterpret<Packet8s>(pcast<Packet8uc, Packet8us>(a));
585 }
586 template <>
587 EIGEN_STRONG_INLINE Packet4s pcast<Packet4uc, Packet4s>(const Packet4uc& a) {
588  return preinterpret<Packet4s>(pcast<Packet4uc, Packet4us>(a));
589 }
590 
591 //==============================================================================
592 // pcast, SrcType = int16_t
593 //==============================================================================
594 template <>
595 struct type_casting_traits<numext::int16_t, float> {
596  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
597 };
598 template <>
599 EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
600  // Discard second half of input.
601  return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
602 }
603 template <>
604 EIGEN_STRONG_INLINE Packet4f pcast<Packet4s, Packet4f>(const Packet4s& a) {
605  return vcvtq_f32_s32(vmovl_s16(a));
606 }
607 template <>
608 EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
609  // Discard second half of input.
610  return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
611 }
612 
613 template <>
614 struct type_casting_traits<numext::int16_t, numext::int64_t> {
615  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
616 };
617 template <>
618 EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
619  // Discard all but first two values.
620  return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
621 }
622 
623 template <>
624 struct type_casting_traits<numext::int16_t, numext::uint64_t> {
625  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
626 };
627 template <>
628 EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
629  return preinterpret<Packet2ul>(pcast<Packet8s, Packet2l>(a));
630 }
631 
632 template <>
633 struct type_casting_traits<numext::int16_t, numext::int32_t> {
634  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
635 };
636 template <>
637 EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
638  // Discard second half of input.
639  return vmovl_s16(vget_low_s16(a));
640 }
641 template <>
642 EIGEN_STRONG_INLINE Packet4i pcast<Packet4s, Packet4i>(const Packet4s& a) {
643  return vmovl_s16(a);
644 }
645 template <>
646 EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
647  // Discard second half of input.
648  return vget_low_s32(vmovl_s16(a));
649 }
650 
651 template <>
652 struct type_casting_traits<numext::int16_t, numext::uint32_t> {
653  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
654 };
655 template <>
656 EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
657  return preinterpret<Packet4ui>(pcast<Packet8s, Packet4i>(a));
658 }
659 template <>
660 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4s, Packet4ui>(const Packet4s& a) {
661  return preinterpret<Packet4ui>(pcast<Packet4s, Packet4i>(a));
662 }
663 template <>
664 EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
665  return preinterpret<Packet2ui>(pcast<Packet4s, Packet2i>(a));
666 }
667 
668 template <>
669 struct type_casting_traits<numext::int16_t, numext::int8_t> {
670  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
671 };
672 template <>
673 EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
674  return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
675 }
676 template <>
677 EIGEN_STRONG_INLINE Packet8c pcast<Packet8s, Packet8c>(const Packet8s& a) {
678  return vmovn_s16(a);
679 }
680 template <>
681 EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
682  return vmovn_s16(vcombine_s16(a, b));
683 }
684 template <>
685 EIGEN_STRONG_INLINE Packet4c pcast<Packet4s, Packet4c>(const Packet4s& a) {
686  const int8x8_t aa_s8x8 = pcast<Packet4s, Packet8c>(a, a);
687  return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
688 }
689 
690 template <>
691 struct type_casting_traits<numext::int16_t, numext::uint8_t> {
692  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
693 };
694 template <>
695 EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
696  return preinterpret<Packet16uc>(pcast<Packet8s, Packet16c>(a, b));
697 }
698 template <>
699 EIGEN_STRONG_INLINE Packet8uc pcast<Packet8s, Packet8uc>(const Packet8s& a) {
700  return preinterpret<Packet8uc>(pcast<Packet8s, Packet8c>(a));
701 }
702 template <>
703 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
704  return preinterpret<Packet8uc>(pcast<Packet4s, Packet8c>(a, b));
705 }
706 template <>
707 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4s, Packet4uc>(const Packet4s& a) {
708  return static_cast<Packet4uc>(pcast<Packet4s, Packet4c>(a));
709 }
710 
711 //==============================================================================
712 // pcast, SrcType = uint16_t
713 //==============================================================================
714 template <>
715 struct type_casting_traits<numext::uint16_t, float> {
716  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
717 };
718 template <>
719 EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
720  // Discard second half of input.
721  return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
722 }
723 template <>
724 EIGEN_STRONG_INLINE Packet4f pcast<Packet4us, Packet4f>(const Packet4us& a) {
725  return vcvtq_f32_u32(vmovl_u16(a));
726 }
727 template <>
728 EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
729  // Discard second half of input.
730  return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
731 }
732 
733 template <>
734 struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
735  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
736 };
737 template <>
738 EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
739  // Discard all but first two values.
740  return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
741 }
742 
743 template <>
744 struct type_casting_traits<numext::uint16_t, numext::int64_t> {
745  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
746 };
747 template <>
748 EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
749  return preinterpret<Packet2l>(pcast<Packet8us, Packet2ul>(a));
750 }
751 
752 template <>
753 struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
754  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
755 };
756 template <>
757 EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
758  // Discard second half of input.
759  return vmovl_u16(vget_low_u16(a));
760 }
761 template <>
762 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4us, Packet4ui>(const Packet4us& a) {
763  return vmovl_u16(a);
764 }
765 template <>
766 EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
767  // Discard second half of input.
768  return vget_low_u32(vmovl_u16(a));
769 }
770 
771 template <>
772 struct type_casting_traits<numext::uint16_t, numext::int32_t> {
773  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
774 };
775 template <>
776 EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
777  return preinterpret<Packet4i>(pcast<Packet8us, Packet4ui>(a));
778 }
779 template <>
780 EIGEN_STRONG_INLINE Packet4i pcast<Packet4us, Packet4i>(const Packet4us& a) {
781  return preinterpret<Packet4i>(pcast<Packet4us, Packet4ui>(a));
782 }
783 template <>
784 EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
785  return preinterpret<Packet2i>(pcast<Packet4us, Packet2ui>(a));
786 }
787 
788 template <>
789 struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
790  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
791 };
792 template <>
793 EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
794  return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
795 }
796 template <>
797 EIGEN_STRONG_INLINE Packet8uc pcast<Packet8us, Packet8uc>(const Packet8us& a) {
798  return vmovn_u16(a);
799 }
800 template <>
801 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
802  return vmovn_u16(vcombine_u16(a, b));
803 }
804 template <>
805 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4us, Packet4uc>(const Packet4us& a) {
806  uint8x8_t aa_u8x8 = pcast<Packet4us, Packet8uc>(a, a);
807  return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
808 }
809 
810 template <>
811 struct type_casting_traits<numext::uint16_t, numext::int8_t> {
812  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
813 };
814 template <>
815 EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
816  return preinterpret<Packet16c>(pcast<Packet8us, Packet16uc>(a, b));
817 }
818 template <>
819 EIGEN_STRONG_INLINE Packet8c pcast<Packet8us, Packet8c>(const Packet8us& a) {
820  return preinterpret<Packet8c>(pcast<Packet8us, Packet8uc>(a));
821 }
822 template <>
823 EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
824  return preinterpret<Packet8c>(pcast<Packet4us, Packet8uc>(a, b));
825 }
826 template <>
827 EIGEN_STRONG_INLINE Packet4c pcast<Packet4us, Packet4c>(const Packet4us& a) {
828  return static_cast<Packet4c>(pcast<Packet4us, Packet4uc>(a));
829 }
830 
831 //==============================================================================
832 // pcast, SrcType = int32_t
833 //==============================================================================
834 template <>
835 struct type_casting_traits<numext::int32_t, float> {
836  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
837 };
838 template <>
839 EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
840  return vcvtq_f32_s32(a);
841 }
842 template <>
843 EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
844  return vcvt_f32_s32(a);
845 }
846 
847 template <>
848 struct type_casting_traits<numext::int32_t, numext::int64_t> {
849  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
850 };
851 template <>
852 EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
853  // Discard second half of input.
854  return vmovl_s32(vget_low_s32(a));
855 }
856 template <>
857 EIGEN_STRONG_INLINE Packet2l pcast<Packet2i, Packet2l>(const Packet2i& a) {
858  return vmovl_s32(a);
859 }
860 
861 template <>
862 struct type_casting_traits<numext::int32_t, numext::uint64_t> {
863  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
864 };
865 template <>
866 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
867  return preinterpret<Packet2ul>(pcast<Packet4i, Packet2l>(a));
868 }
869 template <>
870 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2i, Packet2ul>(const Packet2i& a) {
871  return preinterpret<Packet2ul>(pcast<Packet2i, Packet2l>(a));
872 }
873 
874 template <>
875 struct type_casting_traits<numext::int32_t, numext::int16_t> {
876  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
877 };
878 template <>
879 EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
880  return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
881 }
882 template <>
883 EIGEN_STRONG_INLINE Packet4s pcast<Packet4i, Packet4s>(const Packet4i& a) {
884  return vmovn_s32(a);
885 }
886 template <>
887 EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
888  return vmovn_s32(vcombine_s32(a, b));
889 }
890 
891 template <>
892 struct type_casting_traits<numext::int32_t, numext::uint16_t> {
893  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
894 };
895 template <>
896 EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
897  return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
898 }
899 template <>
900 EIGEN_STRONG_INLINE Packet4us pcast<Packet4i, Packet4us>(const Packet4i& a) {
901  return vmovn_u32(vreinterpretq_u32_s32(a));
902 }
903 template <>
904 EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
905  return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
906 }
907 
908 template <>
909 struct type_casting_traits<numext::int32_t, numext::int8_t> {
910  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
911 };
912 template <>
913 EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
914  const Packet4i& d) {
915  const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
916  const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
917  return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
918 }
919 template <>
920 EIGEN_STRONG_INLINE Packet8c pcast<Packet4i, Packet8c>(const Packet4i& a, const Packet4i& b) {
921  const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
922  return vmovn_s16(ab_s16);
923 }
924 template <>
925 EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
926  const Packet2i& d) {
927  const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
928  const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
929  return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
930 }
931 template <>
932 EIGEN_STRONG_INLINE Packet4c pcast<Packet4i, Packet4c>(const Packet4i& a) {
933  const int16x4_t a_s16x4 = vmovn_s32(a);
934  const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
935  const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
936  return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
937 }
938 
939 template <>
940 struct type_casting_traits<numext::int32_t, numext::uint8_t> {
941  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
942 };
943 template <>
944 EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
945  const Packet4i& d) {
946  return preinterpret<Packet16uc>(pcast<Packet4i, Packet16c>(a, b, c, d));
947 }
948 template <>
949 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4i, Packet8uc>(const Packet4i& a, const Packet4i& b) {
950  return preinterpret<Packet8uc>(pcast<Packet4i, Packet8c>(a, b));
951 }
952 template <>
953 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
954  const Packet2i& d) {
955  return preinterpret<Packet8uc>(pcast<Packet2i, Packet8c>(a, b, c, d));
956 }
957 template <>
958 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4i, Packet4uc>(const Packet4i& a) {
959  return static_cast<Packet4uc>(pcast<Packet4i, Packet4c>(a));
960 }
961 
962 //==============================================================================
963 // pcast, SrcType = uint32_t
964 //==============================================================================
965 template <>
966 struct type_casting_traits<numext::uint32_t, float> {
967  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
968 };
969 template <>
970 EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
971  return vcvtq_f32_u32(a);
972 }
973 template <>
974 EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
975  return vcvt_f32_u32(a);
976 }
977 
978 template <>
979 struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
980  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
981 };
982 template <>
983 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
984  // Discard second half of input.
985  return vmovl_u32(vget_low_u32(a));
986 }
987 template <>
988 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ui, Packet2ul>(const Packet2ui& a) {
989  return vmovl_u32(a);
990 }
991 
992 template <>
993 struct type_casting_traits<numext::uint32_t, numext::int64_t> {
994  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
995 };
996 template <>
997 EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
998  return preinterpret<Packet2l>(pcast<Packet4ui, Packet2ul>(a));
999 }
1000 template <>
1001 EIGEN_STRONG_INLINE Packet2l pcast<Packet2ui, Packet2l>(const Packet2ui& a) {
1002  return preinterpret<Packet2l>(pcast<Packet2ui, Packet2ul>(a));
1003 }
1004 
1005 template <>
1006 struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
1007  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1008 };
1009 template <>
1010 EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
1011  return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1012 }
1013 template <>
1014 EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
1015  return vmovn_u32(vcombine_u32(a, b));
1016 }
1017 template <>
1018 EIGEN_STRONG_INLINE Packet4us pcast<Packet4ui, Packet4us>(const Packet4ui& a) {
1019  return vmovn_u32(a);
1020 }
1021 
1022 template <>
1023 struct type_casting_traits<numext::uint32_t, numext::int16_t> {
1024  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1025 };
1026 template <>
1027 EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
1028  return preinterpret<Packet8s>(pcast<Packet4ui, Packet8us>(a, b));
1029 }
1030 template <>
1031 EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
1032  return preinterpret<Packet4s>(pcast<Packet2ui, Packet4us>(a, b));
1033 }
1034 template <>
1035 EIGEN_STRONG_INLINE Packet4s pcast<Packet4ui, Packet4s>(const Packet4ui& a) {
1036  return preinterpret<Packet4s>(pcast<Packet4ui, Packet4us>(a));
1037 }
1038 
1039 template <>
1040 struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
1041  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1042 };
1043 template <>
1044 EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1045  const Packet4ui& d) {
1046  const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1047  const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
1048  return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
1049 }
1050 template <>
1051 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4ui, Packet8uc>(const Packet4ui& a, const Packet4ui& b) {
1052  const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1053  return vmovn_u16(ab_u16);
1054 }
1055 template <>
1056 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1057  const Packet2ui& d) {
1058  const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
1059  const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
1060  return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
1061 }
1062 template <>
1063 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4ui, Packet4uc>(const Packet4ui& a) {
1064  const uint16x4_t a_u16x4 = vmovn_u32(a);
1065  const uint16x8_t aa_u16x8 = vcombine_u16(a_u16x4, a_u16x4);
1066  const uint8x8_t aa_u8x8 = vmovn_u16(aa_u16x8);
1067  return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
1068 }
1069 
1070 template <>
1071 struct type_casting_traits<numext::uint32_t, numext::int8_t> {
1072  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1073 };
1074 template <>
1075 EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1076  const Packet4ui& d) {
1077  return preinterpret<Packet16c>(pcast<Packet4ui, Packet16uc>(a, b, c, d));
1078 }
1079 template <>
1080 EIGEN_STRONG_INLINE Packet8c pcast<Packet4ui, Packet8c>(const Packet4ui& a, const Packet4ui& b) {
1081  return preinterpret<Packet8c>(pcast<Packet4ui, Packet8uc>(a, b));
1082 }
1083 template <>
1084 EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1085  const Packet2ui& d) {
1086  return preinterpret<Packet8c>(pcast<Packet2ui, Packet8uc>(a, b, c, d));
1087 }
1088 template <>
1089 EIGEN_STRONG_INLINE Packet4c pcast<Packet4ui, Packet4c>(const Packet4ui& a) {
1090  return static_cast<Packet4c>(pcast<Packet4ui, Packet4uc>(a));
1091 }
1092 
1093 //==============================================================================
1094 // pcast, SrcType = int64_t
1095 //==============================================================================
1096 template <>
1097 struct type_casting_traits<numext::int64_t, float> {
1098  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1099 };
1100 
1101 template <>
1102 EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
1103 #if EIGEN_ARCH_ARM64
1104  return vcombine_f32(vcvt_f32_f64(vcvtq_f64_s64(a)), vcvt_f32_f64(vcvtq_f64_s64(b)));
1105 #else
1106  EIGEN_ALIGN_MAX int64_t lvals[4];
1107  pstore(lvals, a);
1108  pstore(lvals + 2, b);
1109  EIGEN_ALIGN_MAX float fvals[4] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1]),
1110  static_cast<float>(lvals[2]), static_cast<float>(lvals[3])};
1111  return pload<Packet4f>(fvals);
1112 #endif
1113 }
1114 
1115 template <>
1116 EIGEN_STRONG_INLINE Packet2f pcast<Packet2l, Packet2f>(const Packet2l& a) {
1117 #if EIGEN_ARCH_ARM64
1118  return vcvt_f32_f64(vcvtq_f64_s64(a));
1119 #else
1120  EIGEN_ALIGN_MAX int64_t lvals[2];
1121  pstore(lvals, a);
1122  EIGEN_ALIGN_MAX float fvals[2] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1])};
1123  return pload<Packet2f>(fvals);
1124 #endif
1125 }
1126 
1127 template <>
1128 struct type_casting_traits<numext::int64_t, numext::int32_t> {
1129  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1130 };
1131 template <>
1132 EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
1133  return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
1134 }
1135 template <>
1136 EIGEN_STRONG_INLINE Packet2i pcast<Packet2l, Packet2i>(const Packet2l& a) {
1137  return vmovn_s64(a);
1138 }
1139 
1140 template <>
1141 struct type_casting_traits<numext::int64_t, numext::uint32_t> {
1142  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1143 };
1144 template <>
1145 EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
1146  return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
1147 }
1148 template <>
1149 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2l, Packet2ui>(const Packet2l& a) {
1150  return vmovn_u64(vreinterpretq_u64_s64(a));
1151 }
1152 
1153 template <>
1154 struct type_casting_traits<numext::int64_t, numext::int16_t> {
1155  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1156 };
1157 template <>
1158 EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1159  const Packet2l& d) {
1160  const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1161  const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
1162  return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1163 }
1164 template <>
1165 EIGEN_STRONG_INLINE Packet4s pcast<Packet2l, Packet4s>(const Packet2l& a, const Packet2l& b) {
1166  const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1167  return vmovn_s32(ab_s32);
1168 }
1169 
1170 template <>
1171 struct type_casting_traits<numext::int64_t, numext::uint16_t> {
1172  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1173 };
1174 template <>
1175 EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1176  const Packet2l& d) {
1177  return preinterpret<Packet8us>(pcast<Packet2l, Packet8s>(a, b, c, d));
1178 }
1179 template <>
1180 EIGEN_STRONG_INLINE Packet4us pcast<Packet2l, Packet4us>(const Packet2l& a, const Packet2l& b) {
1181  return preinterpret<Packet4us>(pcast<Packet2l, Packet4s>(a, b));
1182 }
1183 
1184 template <>
1185 struct type_casting_traits<numext::int64_t, numext::int8_t> {
1186  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1187 };
1188 template <>
1189 EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1190  const Packet2l& d, const Packet2l& e, const Packet2l& f,
1191  const Packet2l& g, const Packet2l& h) {
1192  const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1193  const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
1194  return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1195 }
1196 template <>
1197 EIGEN_STRONG_INLINE Packet8c pcast<Packet2l, Packet8c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1198  const Packet2l& d) {
1199  const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1200  return vmovn_s16(abcd_s16);
1201 }
1202 template <>
1203 EIGEN_STRONG_INLINE Packet4c pcast<Packet2l, Packet4c>(const Packet2l& a, const Packet2l& b) {
1204  const int16x4_t ab_s16 = pcast<Packet2l, Packet4s>(a, b);
1205  const int16x8_t abab_s16 = vcombine_s16(ab_s16, ab_s16);
1206  const int8x8_t abab_s8 = vmovn_s16(abab_s16);
1207  return vget_lane_s32(vreinterpret_s32_s8(abab_s8), 0);
1208 }
1209 
1210 template <>
1211 struct type_casting_traits<numext::int64_t, numext::uint8_t> {
1212  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1213 };
1214 template <>
1215 EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1216  const Packet2l& d, const Packet2l& e, const Packet2l& f,
1217  const Packet2l& g, const Packet2l& h) {
1218  const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
1219  const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
1220  return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1221 }
1222 template <>
1223 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2l, Packet8uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1224  const Packet2l& d) {
1225  return preinterpret<Packet8uc>(pcast<Packet2l, Packet8c>(a, b, c, d));
1226 }
1227 template <>
1228 EIGEN_STRONG_INLINE Packet4uc pcast<Packet2l, Packet4uc>(const Packet2l& a, const Packet2l& b) {
1229  return static_cast<Packet4uc>(pcast<Packet2l, Packet4c>(a, b));
1230 }
1231 
1232 //==============================================================================
1233 // pcast, SrcType = uint64_t
1234 //==============================================================================
1235 template <>
1236 struct type_casting_traits<numext::uint64_t, float> {
1237  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1238 };
1239 template <>
1240 EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
1241 #if EIGEN_ARCH_ARM64
1242  return vcombine_f32(vcvt_f32_f64(vcvtq_f64_u64(a)), vcvt_f32_f64(vcvtq_f64_u64(b)));
1243 #else
1244  EIGEN_ALIGN_MAX uint64_t uvals[4];
1245  pstore(uvals, a);
1246  pstore(uvals + 2, b);
1247  EIGEN_ALIGN_MAX float fvals[4] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1]),
1248  static_cast<float>(uvals[2]), static_cast<float>(uvals[3])};
1249  return pload<Packet4f>(fvals);
1250 #endif
1251 }
1252 template <>
1253 EIGEN_STRONG_INLINE Packet2f pcast<Packet2ul, Packet2f>(const Packet2ul& a) {
1254 #if EIGEN_ARCH_ARM64
1255  return vcvt_f32_f64(vcvtq_f64_u64(a));
1256 #else
1257  EIGEN_ALIGN_MAX uint64_t uvals[2];
1258  pstore(uvals, a);
1259  EIGEN_ALIGN_MAX float fvals[2] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1])};
1260  return pload<Packet2f>(fvals);
1261 #endif
1262 }
1263 
1264 template <>
1265 struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1266  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1267 };
1268 template <>
1269 EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1270  return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1271 }
1272 template <>
1273 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ul, Packet2ui>(const Packet2ul& a) {
1274  return vmovn_u64(a);
1275 }
1276 
1277 template <>
1278 struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1279  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1280 };
1281 template <>
1282 EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1283  return preinterpret<Packet4i>(pcast<Packet2ul, Packet4ui>(a, b));
1284 }
1285 template <>
1286 EIGEN_STRONG_INLINE Packet2i pcast<Packet2ul, Packet2i>(const Packet2ul& a) {
1287  return preinterpret<Packet2i>(pcast<Packet2ul, Packet2ui>(a));
1288 }
1289 
1290 template <>
1291 struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1292  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1293 };
1294 template <>
1295 EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1296  const Packet2ul& d) {
1297  const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1298  const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1299  return vcombine_u16(ab_u16, cd_u16);
1300 }
1301 template <>
1302 EIGEN_STRONG_INLINE Packet4us pcast<Packet2ul, Packet4us>(const Packet2ul& a, const Packet2ul& b) {
1303  return vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1304 }
1305 
1306 template <>
1307 struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1308  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1309 };
1310 template <>
1311 EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1312  const Packet2ul& d) {
1313  return preinterpret<Packet8s>(pcast<Packet2ul, Packet8us>(a, b, c, d));
1314 }
1315 template <>
1316 EIGEN_STRONG_INLINE Packet4s pcast<Packet2ul, Packet4s>(const Packet2ul& a, const Packet2ul& b) {
1317  return preinterpret<Packet4s>(pcast<Packet2ul, Packet4us>(a, b));
1318 }
1319 
1320 template <>
1321 struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1322  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1323 };
1324 template <>
1325 EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1326  const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1327  const Packet2ul& g, const Packet2ul& h) {
1328  const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1329  const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1330  return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1331 }
1332 template <>
1333 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ul, Packet8uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1334  const Packet2ul& d) {
1335  const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1336  return vmovn_u16(abcd_u16);
1337 }
1338 template <>
1339 EIGEN_STRONG_INLINE Packet4uc pcast<Packet2ul, Packet4uc>(const Packet2ul& a, const Packet2ul& b) {
1340  const uint16x4_t ab_u16 = pcast<Packet2ul, Packet4us>(a, b);
1341  const uint16x8_t abab_u16 = vcombine_u16(ab_u16, ab_u16);
1342  const uint8x8_t abab_u8 = vmovn_u16(abab_u16);
1343  return vget_lane_u32(vreinterpret_u32_u8(abab_u8), 0);
1344 }
1345 
1346 template <>
1347 struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1348  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1349 };
1350 template <>
1351 EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1352  const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1353  const Packet2ul& g, const Packet2ul& h) {
1354  return preinterpret<Packet16c>(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1355 }
1356 template <>
1357 EIGEN_STRONG_INLINE Packet8c pcast<Packet2ul, Packet8c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1358  const Packet2ul& d) {
1359  return preinterpret<Packet8c>(pcast<Packet2ul, Packet8uc>(a, b, c, d));
1360 }
1361 template <>
1362 EIGEN_STRONG_INLINE Packet4c pcast<Packet2ul, Packet4c>(const Packet2ul& a, const Packet2ul& b) {
1363  return static_cast<Packet4c>(pcast<Packet2ul, Packet4uc>(a, b));
1364 }
1365 
1366 #if EIGEN_ARCH_ARM64
1367 
1368 //==============================================================================
1369 // pcast/preinterpret, Double
1370 //==============================================================================
1371 
1372 template <>
1373 EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1374  return Packet2d(vreinterpretq_f64_s64(a));
1375 }
1376 template <>
1377 EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1378  return Packet2d(vreinterpretq_f64_u64(a));
1379 }
1380 template <>
1381 EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1382  return Packet2l(vreinterpretq_s64_f64(a));
1383 }
1384 template <>
1385 EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1386  return Packet2ul(vreinterpretq_u64_f64(a));
1387 }
1388 template <>
1389 EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1390  return Packet2d(vreinterpretq_f64_s32(a));
1391 }
1392 template <>
1393 EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1394  return Packet4i(vreinterpretq_s32_f64(a));
1395 }
1396 
1397 template <>
1398 struct type_casting_traits<double, float> {
1399  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1400 };
1401 template <>
1402 EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1403  return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1404 }
1405 template <>
1406 EIGEN_STRONG_INLINE Packet2f pcast<Packet2d, Packet2f>(const Packet2d& a) {
1407  return vcvt_f32_f64(a);
1408 }
1409 
1410 template <>
1411 struct type_casting_traits<double, numext::int64_t> {
1412  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1413 };
1414 template <>
1415 EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1416  return vcvtq_s64_f64(a);
1417 }
1418 
1419 template <>
1420 struct type_casting_traits<double, numext::uint64_t> {
1421  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1422 };
1423 template <>
1424 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1425  return vcvtq_u64_f64(a);
1426 }
1427 
1428 template <>
1429 struct type_casting_traits<double, numext::int32_t> {
1430  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1431 };
1432 template <>
1433 EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1434  return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1435 }
1436 template <>
1437 EIGEN_STRONG_INLINE Packet2i pcast<Packet2d, Packet2i>(const Packet2d& a) {
1438  return vmovn_s64(vcvtq_s64_f64(a));
1439 }
1440 
1441 template <>
1442 struct type_casting_traits<double, numext::uint32_t> {
1443  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1444 };
1445 template <>
1446 EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1447  return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1448 }
1449 template <>
1450 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2d, Packet2ui>(const Packet2d& a) {
1451  return vmovn_u64(vcvtq_u64_f64(a));
1452 }
1453 
1454 template <>
1455 struct type_casting_traits<double, numext::int16_t> {
1456  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1457 };
1458 template <>
1459 EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1460  const Packet2d& d) {
1461  const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1462  const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1463  return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1464 }
1465 template <>
1466 EIGEN_STRONG_INLINE Packet4s pcast<Packet2d, Packet4s>(const Packet2d& a, const Packet2d& b) {
1467  const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1468  return vmovn_s32(ab_s32);
1469 }
1470 
1471 template <>
1472 struct type_casting_traits<double, numext::uint16_t> {
1473  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1474 };
1475 template <>
1476 EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1477  const Packet2d& d) {
1478  return preinterpret<Packet8us>(pcast<Packet2d, Packet8s>(a, b, c, d));
1479 }
1480 template <>
1481 EIGEN_STRONG_INLINE Packet4us pcast<Packet2d, Packet4us>(const Packet2d& a, const Packet2d& b) {
1482  return preinterpret<Packet4us>(pcast<Packet2d, Packet4s>(a, b));
1483 }
1484 
1485 template <>
1486 struct type_casting_traits<double, numext::int8_t> {
1487  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1488 };
1489 template <>
1490 EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1491  const Packet2d& d, const Packet2d& e, const Packet2d& f,
1492  const Packet2d& g, const Packet2d& h) {
1493  const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1494  const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1495  return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1496 }
1497 template <>
1498 EIGEN_STRONG_INLINE Packet8c pcast<Packet2d, Packet8c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1499  const Packet2d& d) {
1500  const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1501  return vmovn_s16(abcd_s16);
1502 }
1503 template <>
1504 EIGEN_STRONG_INLINE Packet4c pcast<Packet2d, Packet4c>(const Packet2d& a, const Packet2d& b) {
1505  const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1506  return pcast<Packet4i, Packet4c>(ab_s32);
1507 }
1508 
1509 template <>
1510 struct type_casting_traits<double, numext::uint8_t> {
1511  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1512 };
1513 template <>
1514 EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1515  const Packet2d& d, const Packet2d& e, const Packet2d& f,
1516  const Packet2d& g, const Packet2d& h) {
1517  const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1518  const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1519  return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1520 }
1521 template <>
1522 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2d, Packet8uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1523  const Packet2d& d) {
1524  return preinterpret<Packet8uc>(pcast<Packet2d, Packet8c>(a, b, c, d));
1525 }
1526 template <>
1527 EIGEN_STRONG_INLINE Packet4uc pcast<Packet2d, Packet4uc>(const Packet2d& a, const Packet2d& b) {
1528  return static_cast<Packet4uc>(pcast<Packet2d, Packet4c>(a, b));
1529 }
1530 
1531 template <>
1532 struct type_casting_traits<float, double> {
1533  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1534 };
1535 template <>
1536 EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1537  // Discard second-half of input.
1538  return vcvt_f64_f32(vget_low_f32(a));
1539 }
1540 template <>
1541 EIGEN_STRONG_INLINE Packet2d pcast<Packet2f, Packet2d>(const Packet2f& a) {
1542  return vcvt_f64_f32(a);
1543 }
1544 
1545 template <>
1546 struct type_casting_traits<numext::int8_t, double> {
1547  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1548 };
1549 template <>
1550 EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1551  // Discard all but first two values.
1552  // MSVC defines most intrinsics as macros, so we need to do this in two lines for portability.
1553  Packet2f tmp = pcast<Packet8c, Packet2f>(vget_low_s8(a));
1554  return vcvt_f64_f32(tmp);
1555 }
1556 
1557 template <>
1558 struct type_casting_traits<numext::uint8_t, double> {
1559  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1560 };
1561 template <>
1562 EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1563  // Discard all but first two values.
1564  Packet2f tmp = pcast<Packet8uc, Packet2f>(vget_low_u8(a));
1565  return vcvt_f64_f32(tmp);
1566 }
1567 
1568 template <>
1569 struct type_casting_traits<numext::int16_t, double> {
1570  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1571 };
1572 template <>
1573 EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1574  // Discard all but first two values.
1575  Packet2f tmp = pcast<Packet4s, Packet2f>(vget_low_s16(a));
1576  return vcvt_f64_f32(tmp);
1577 }
1578 
1579 template <>
1580 struct type_casting_traits<numext::uint16_t, double> {
1581  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1582 };
1583 template <>
1584 EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1585  // Discard all but first two values.
1586  Packet2f tmp = pcast<Packet4us, Packet2f>(vget_low_u16(a));
1587  return vcvt_f64_f32(tmp);
1588 }
1589 
1590 template <>
1591 struct type_casting_traits<numext::int32_t, double> {
1592  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1593 };
1594 template <>
1595 EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1596  // Discard second half of input.
1597  return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1598 }
1599 template <>
1600 EIGEN_STRONG_INLINE Packet2d pcast<Packet2i, Packet2d>(const Packet2i& a) {
1601  return vcvtq_f64_s64(vmovl_s32(a));
1602 }
1603 
1604 template <>
1605 struct type_casting_traits<numext::uint32_t, double> {
1606  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1607 };
1608 template <>
1609 EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1610  // Discard second half of input.
1611  return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1612 }
1613 template <>
1614 EIGEN_STRONG_INLINE Packet2d pcast<Packet2ui, Packet2d>(const Packet2ui& a) {
1615  return vcvtq_f64_u64(vmovl_u32(a));
1616 }
1617 
1618 template <>
1619 struct type_casting_traits<numext::int64_t, double> {
1620  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1621 };
1622 template <>
1623 EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1624  return vcvtq_f64_s64(a);
1625 }
1626 
1627 template <>
1628 struct type_casting_traits<numext::uint64_t, double> {
1629  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1630 };
1631 template <>
1632 EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1633  return vcvtq_f64_u64(a);
1634 }
1635 
1636 #endif // EIGEN_ARCH_ARM64
1637 
1638 } // end namespace internal
1639 
1640 } // end namespace Eigen
1641 
1642 #endif // EIGEN_TYPE_CASTING_NEON_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition: B01_Experimental.dox:1
Generated by doxygen 1.8.14