$darkmode
Eigen  5.0.1-dev
PacketMath.h
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5 //
6 // This Source Code Form is subject to the terms of the Mozilla
7 // Public License v. 2.0. If a copy of the MPL was not distributed
8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 
10 #ifndef EIGEN_PACKET_MATH_AVX512_H
11 #define EIGEN_PACKET_MATH_AVX512_H
12 
13 // IWYU pragma: private
14 #include "../../InternalHeaderCheck.h"
15 
16 namespace Eigen {
17 
18 namespace internal {
19 
20 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
21 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
22 #endif
23 
24 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
25 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
26 #endif
27 
28 #ifdef EIGEN_VECTORIZE_FMA
29 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
30 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
31 #endif
32 #endif
33 
34 typedef __m512 Packet16f;
35 typedef __m512i Packet16i;
36 typedef __m512d Packet8d;
37 typedef eigen_packet_wrapper<__m512i, 1> Packet8l;
38 #ifndef EIGEN_VECTORIZE_AVX512FP16
39 typedef eigen_packet_wrapper<__m256i, 1> Packet16h;
40 #endif
41 typedef eigen_packet_wrapper<__m256i, 2> Packet16bf;
42 
43 typedef eigen_packet_wrapper<__m512i, 6> Packet32s;
44 typedef eigen_packet_wrapper<__m256i, 6> Packet16s;
45 typedef eigen_packet_wrapper<__m128i, 6> Packet8s;
46 
47 template <>
48 struct is_arithmetic<__m512> {
49  enum { value = true };
50 };
51 template <>
52 struct is_arithmetic<__m512i> {
53  enum { value = true };
54 };
55 template <>
56 struct is_arithmetic<__m512d> {
57  enum { value = true };
58 };
59 template <>
60 struct is_arithmetic<Packet8l> {
61  enum { value = true };
62 };
63 
64 #ifndef EIGEN_VECTORIZE_AVX512FP16
65 template <>
66 struct is_arithmetic<Packet16h> {
67  enum { value = true };
68 };
69 
70 template <>
71 struct packet_traits<half> : default_packet_traits {
72  typedef Packet16h type;
73  // There is no half-size packet for Packet16h.
74  typedef Packet16h half;
75  enum {
76  Vectorizable = 1,
77  AlignedOnScalar = 1,
78  size = 16,
79 
80  HasCmp = 1,
81  HasAdd = 1,
82  HasSub = 1,
83  HasMul = 1,
84  HasDiv = 1,
85  HasNegate = 1,
86  HasAbs = 1,
87  HasAbs2 = 0,
88  HasMin = 1,
89  HasMax = 1,
90  HasConj = 1,
91  HasSetLinear = 0,
92  HasSqrt = 1,
93  HasRsqrt = 1,
94  HasLog = 1,
95  HasLog1p = 1,
96  HasExp = 1,
97  HasExpm1 = 1,
98  HasBessel = 1,
99  HasNdtri = 1,
100  HasSin = EIGEN_FAST_MATH,
101  HasCos = EIGEN_FAST_MATH,
102  HasTanh = EIGEN_FAST_MATH,
103  HasErf = EIGEN_FAST_MATH,
104  HasBlend = 0
105  };
106 };
107 #endif
108 
109 template <>
110 struct packet_traits<float> : default_packet_traits {
111  typedef Packet16f type;
112  typedef Packet8f half;
113  enum {
114  Vectorizable = 1,
115  AlignedOnScalar = 1,
116  size = 16,
117 
118  HasAbs = 1,
119  HasMin = 1,
120  HasMax = 1,
121  HasConj = 1,
122  HasBlend = 1,
123  HasSin = EIGEN_FAST_MATH,
124  HasCos = EIGEN_FAST_MATH,
125  HasACos = 1,
126  HasASin = 1,
127  HasATan = 1,
128  HasATanh = 1,
129  HasSqrt = 1,
130  HasRsqrt = 1,
131  HasCbrt = 1,
132  HasLog = 1,
133  HasLog1p = 1,
134  HasExpm1 = 1,
135  HasNdtri = 1,
136  HasBessel = 1,
137  HasExp = 1,
138  HasPow = 1,
139  HasReciprocal = EIGEN_FAST_MATH,
140  HasTanh = EIGEN_FAST_MATH,
141  HasErf = EIGEN_FAST_MATH,
142  HasErfc = EIGEN_FAST_MATH,
143  HasCmp = 1,
144  HasDiv = 1
145  };
146 };
147 template <>
148 struct packet_traits<double> : default_packet_traits {
149  typedef Packet8d type;
150  typedef Packet4d half;
151  enum {
152  Vectorizable = 1,
153  AlignedOnScalar = 1,
154  size = 8,
155  HasBlend = 1,
156  HasSqrt = 1,
157  HasRsqrt = 1,
158  HasCbrt = 1,
159  HasSin = EIGEN_FAST_MATH,
160  HasCos = EIGEN_FAST_MATH,
161  HasLog = 1,
162  HasExp = 1,
163  HasPow = 1,
164  HasATan = 1,
165  HasTanh = EIGEN_FAST_MATH,
166  HasErf = EIGEN_FAST_MATH,
167  HasErfc = EIGEN_FAST_MATH,
168  HasATanh = 1,
169  HasCmp = 1,
170  HasDiv = 1
171  };
172 };
173 
174 template <>
175 struct packet_traits<int> : default_packet_traits {
176  typedef Packet16i type;
177  typedef Packet8i half;
178  enum { Vectorizable = 1, AlignedOnScalar = 1, HasBlend = 0, HasCmp = 1, HasDiv = 1, size = 16 };
179 };
180 
181 template <>
182 struct packet_traits<int64_t> : default_packet_traits {
183  typedef Packet8l type;
184  typedef Packet4l half;
185  enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, size = 8 };
186 };
187 
188 template <>
189 struct unpacket_traits<Packet16f> {
190  typedef float type;
191  typedef Packet8f half;
192  typedef Packet16i integer_packet;
193  typedef uint16_t mask_t;
194  enum {
195  size = 16,
196  alignment = Aligned64,
197  vectorizable = true,
198  masked_load_available = true,
199  masked_store_available = true,
200  masked_fpops_available = true
201  };
202 };
203 template <>
204 struct unpacket_traits<Packet8d> {
205  typedef double type;
206  typedef Packet4d half;
207  typedef Packet8l integer_packet;
208  typedef uint8_t mask_t;
209  enum {
210  size = 8,
211  alignment = Aligned64,
212  vectorizable = true,
213  masked_load_available = true,
214  masked_store_available = true,
215  masked_fpops_available = true
216  };
217 };
218 template <>
219 struct unpacket_traits<Packet16i> {
220  typedef int type;
221  typedef Packet8i half;
222  enum {
223  size = 16,
224  alignment = Aligned64,
225  vectorizable = true,
226  masked_load_available = false,
227  masked_store_available = false
228  };
229 };
230 
231 template <>
232 struct unpacket_traits<Packet8l> {
233  typedef int64_t type;
234  typedef Packet4l half;
235  enum {
236  size = 8,
237  alignment = Aligned64,
238  vectorizable = true,
239  masked_load_available = false,
240  masked_store_available = false
241  };
242 };
243 
244 #ifndef EIGEN_VECTORIZE_AVX512FP16
245 template <>
246 struct unpacket_traits<Packet16h> {
247  typedef Eigen::half type;
248  typedef Packet8h half;
249  enum {
250  size = 16,
251  alignment = Aligned32,
252  vectorizable = true,
253  masked_load_available = false,
254  masked_store_available = false
255  };
256 };
257 #endif
258 
259 template <>
260 struct unpacket_traits<Packet32s> {
261  typedef numext::int16_t type;
262  typedef Packet16s half;
263  enum {
264  size = 32,
265  alignment = Aligned64,
266  vectorizable = false,
267  };
268 };
269 
270 template <>
271 struct unpacket_traits<Packet16s> {
272  typedef numext::int16_t type;
273  typedef Packet8s half;
274  enum {
275  size = 16,
276  alignment = Aligned32,
277  vectorizable = false,
278  };
279 };
280 
281 template <>
282 struct unpacket_traits<Packet8s> {
283  typedef numext::int16_t type;
284  typedef Packet8s half;
285  enum {
286  size = 8,
287  alignment = Aligned16,
288  vectorizable = false,
289  };
290 };
291 
292 template <>
293 EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
294  return _mm512_set1_ps(from);
295 }
296 template <>
297 EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
298  return _mm512_set1_pd(from);
299 }
300 template <>
301 EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
302  return _mm512_set1_epi32(from);
303 }
304 template <>
305 EIGEN_STRONG_INLINE Packet8l pset1<Packet8l>(const int64_t& from) {
306  return _mm512_set1_epi64(from);
307 }
308 
309 template <>
310 EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(unsigned int from) {
311  return _mm512_castsi512_ps(_mm512_set1_epi32(from));
312 }
313 
314 template <>
315 EIGEN_STRONG_INLINE Packet8d pset1frombits<Packet8d>(const numext::uint64_t from) {
316  return _mm512_castsi512_pd(_mm512_set1_epi64(from));
317 }
318 
319 template <>
320 EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f& /*a*/) {
321  return _mm512_setzero_ps();
322 }
323 template <>
324 EIGEN_STRONG_INLINE Packet8d pzero(const Packet8d& /*a*/) {
325  return _mm512_setzero_pd();
326 }
327 template <>
328 EIGEN_STRONG_INLINE Packet16i pzero(const Packet16i& /*a*/) {
329  return _mm512_setzero_si512();
330 }
331 
332 template <>
333 EIGEN_STRONG_INLINE Packet8l pzero(const Packet8l& /*a*/) {
334  return _mm512_setzero_si512();
335 }
336 
337 template <>
338 EIGEN_STRONG_INLINE Packet16f peven_mask(const Packet16f& /*a*/) {
339  return _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1));
340 }
341 template <>
342 EIGEN_STRONG_INLINE Packet16i peven_mask(const Packet16i& /*a*/) {
343  return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1);
344 }
345 template <>
346 EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) {
347  return _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1));
348 }
349 template <>
350 EIGEN_STRONG_INLINE Packet8l peven_mask(const Packet8l& /*a*/) {
351  return _mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1);
352 }
353 
354 template <>
355 EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
356 #if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
357  // Inline asm here helps reduce some register spilling in TRSM kernels.
358  // See note in unrolls::gemm::microKernel in TrsmKernel.h
359  Packet16f ret;
360  __asm__("vbroadcastss %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
361  return ret;
362 #else
363  return _mm512_broadcastss_ps(_mm_load_ps1(from));
364 #endif
365 }
366 template <>
367 EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
368 #if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
369  Packet8d ret;
370  __asm__("vbroadcastsd %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
371  return ret;
372 #else
373  return _mm512_set1_pd(*from);
374 #endif
375 }
376 
377 template <>
378 EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
379  return _mm512_add_ps(_mm512_set1_ps(a), _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f,
380  6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
381 }
382 template <>
383 EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
384  return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
385 }
386 template <>
387 EIGEN_STRONG_INLINE Packet16i plset<Packet16i>(const int& a) {
388  return _mm512_add_epi32(_mm512_set1_epi32(a), _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
389 }
390 template <>
391 EIGEN_STRONG_INLINE Packet8l plset<Packet8l>(const int64_t& a) {
392  return _mm512_add_epi64(_mm512_set1_epi64(a), _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0));
393 }
394 
395 template <>
396 EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b) {
397  return _mm512_add_ps(a, b);
398 }
399 template <>
400 EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, const Packet8d& b) {
401  return _mm512_add_pd(a, b);
402 }
403 template <>
404 EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a, const Packet16i& b) {
405  return _mm512_add_epi32(a, b);
406 }
407 template <>
408 EIGEN_STRONG_INLINE Packet8l padd<Packet8l>(const Packet8l& a, const Packet8l& b) {
409  return _mm512_add_epi64(a, b);
410 }
411 
412 template <>
413 EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b, uint16_t umask) {
414  __mmask16 mask = static_cast<__mmask16>(umask);
415  return _mm512_maskz_add_ps(mask, a, b);
416 }
417 template <>
418 EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, const Packet8d& b, uint8_t umask) {
419  __mmask8 mask = static_cast<__mmask8>(umask);
420  return _mm512_maskz_add_pd(mask, a, b);
421 }
422 
423 template <>
424 EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a, const Packet16f& b) {
425  return _mm512_sub_ps(a, b);
426 }
427 template <>
428 EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a, const Packet8d& b) {
429  return _mm512_sub_pd(a, b);
430 }
431 template <>
432 EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a, const Packet16i& b) {
433  return _mm512_sub_epi32(a, b);
434 }
435 template <>
436 EIGEN_STRONG_INLINE Packet8l psub<Packet8l>(const Packet8l& a, const Packet8l& b) {
437  return _mm512_sub_epi64(a, b);
438 }
439 
440 template <>
441 EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
442  // NOTE: MSVC seems to struggle with _mm512_set1_epi32, leading to random results.
443  // The intel docs give it a relatively high latency as well, so we're probably
444  // better off with using _mm512_set_epi32 directly anyways.
445  const __m512i mask =
446  _mm512_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000,
447  0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000);
448  return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask));
449 }
450 template <>
451 EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
452  const __m512i mask =
453  _mm512_set_epi64(0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL,
454  0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL);
455  return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask));
456 }
457 template <>
458 EIGEN_STRONG_INLINE Packet16i pnegate(const Packet16i& a) {
459  return _mm512_sub_epi32(_mm512_setzero_si512(), a);
460 }
461 template <>
462 EIGEN_STRONG_INLINE Packet8l pnegate(const Packet8l& a) {
463  return _mm512_sub_epi64(_mm512_setzero_si512(), a);
464 }
465 
466 template <>
467 EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) {
468  return a;
469 }
470 template <>
471 EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) {
472  return a;
473 }
474 template <>
475 EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) {
476  return a;
477 }
478 template <>
479 EIGEN_STRONG_INLINE Packet8l pconj(const Packet8l& a) {
480  return a;
481 }
482 
483 template <>
484 EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a, const Packet16f& b) {
485  return _mm512_mul_ps(a, b);
486 }
487 template <>
488 EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a, const Packet8d& b) {
489  return _mm512_mul_pd(a, b);
490 }
491 template <>
492 EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a, const Packet16i& b) {
493  return _mm512_mullo_epi32(a, b);
494 }
495 template <>
496 EIGEN_STRONG_INLINE Packet8l pmul<Packet8l>(const Packet8l& a, const Packet8l& b) {
497 #ifdef EIGEN_VECTORIZE_AVX512DQ
498  return _mm512_mullo_epi64(a, b);
499 #else
500  return _mm512_mullox_epi64(a, b);
501 #endif
502 }
503 
504 template <>
505 EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a, const Packet16f& b) {
506  return _mm512_div_ps(a, b);
507 }
508 
509 template <>
510 EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a, const Packet8d& b) {
511  return _mm512_div_pd(a, b);
512 }
513 
514 template <>
515 EIGEN_STRONG_INLINE Packet16i pdiv<Packet16i>(const Packet16i& a, const Packet16i& b) {
516  Packet8i q_lo = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b, 0));
517  Packet8i q_hi = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1));
518  return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1);
519 }
520 
521 #ifdef EIGEN_VECTORIZE_FMA
522 template <>
523 EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
524  return _mm512_fmadd_ps(a, b, c);
525 }
526 template <>
527 EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
528  return _mm512_fmadd_pd(a, b, c);
529 }
530 
531 template <>
532 EIGEN_STRONG_INLINE Packet16f pmsub(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
533  return _mm512_fmsub_ps(a, b, c);
534 }
535 template <>
536 EIGEN_STRONG_INLINE Packet8d pmsub(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
537  return _mm512_fmsub_pd(a, b, c);
538 }
539 
540 template <>
541 EIGEN_STRONG_INLINE Packet16f pnmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
542  return _mm512_fnmadd_ps(a, b, c);
543 }
544 template <>
545 EIGEN_STRONG_INLINE Packet8d pnmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
546  return _mm512_fnmadd_pd(a, b, c);
547 }
548 
549 template <>
550 EIGEN_STRONG_INLINE Packet16f pnmsub(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
551  return _mm512_fnmsub_ps(a, b, c);
552 }
553 template <>
554 EIGEN_STRONG_INLINE Packet8d pnmsub(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
555  return _mm512_fnmsub_pd(a, b, c);
556 }
557 #endif
558 
559 template <>
560 EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) {
561  __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32());
562  return _mm512_mask_blend_ps(mask16, a, b);
563 }
564 
565 template <>
566 EIGEN_DEVICE_FUNC inline Packet16i pselect(const Packet16i& mask, const Packet16i& a, const Packet16i& b) {
567  __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32());
568  return _mm512_mask_blend_epi32(mask16, a, b);
569 }
570 
571 template <>
572 EIGEN_DEVICE_FUNC inline Packet8l pselect(const Packet8l& mask, const Packet8l& a, const Packet8l& b) {
573  __mmask8 mask8 = _mm512_cmpeq_epi64_mask(mask, _mm512_setzero_si512());
574  return _mm512_mask_blend_epi64(mask8, a, b);
575 }
576 
577 template <>
578 EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask, const Packet8d& a, const Packet8d& b) {
579  __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
580  return _mm512_mask_blend_pd(mask8, a, b);
581 }
582 
583 template <>
584 EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a, const Packet16f& b) {
585  // Arguments are reversed to match NaN propagation behavior of std::min.
586  return _mm512_min_ps(b, a);
587 }
588 template <>
589 EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a, const Packet8d& b) {
590  // Arguments are reversed to match NaN propagation behavior of std::min.
591  return _mm512_min_pd(b, a);
592 }
593 template <>
594 EIGEN_STRONG_INLINE Packet16i pmin<Packet16i>(const Packet16i& a, const Packet16i& b) {
595  return _mm512_min_epi32(b, a);
596 }
597 template <>
598 EIGEN_STRONG_INLINE Packet8l pmin<Packet8l>(const Packet8l& a, const Packet8l& b) {
599  return _mm512_min_epi64(b, a);
600 }
601 
602 template <>
603 EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a, const Packet16f& b) {
604  // Arguments are reversed to match NaN propagation behavior of std::max.
605  return _mm512_max_ps(b, a);
606 }
607 template <>
608 EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a, const Packet8d& b) {
609  // Arguments are reversed to match NaN propagation behavior of std::max.
610  return _mm512_max_pd(b, a);
611 }
612 template <>
613 EIGEN_STRONG_INLINE Packet16i pmax<Packet16i>(const Packet16i& a, const Packet16i& b) {
614  return _mm512_max_epi32(b, a);
615 }
616 template <>
617 EIGEN_STRONG_INLINE Packet8l pmax<Packet8l>(const Packet8l& a, const Packet8l& b) {
618  return _mm512_max_epi64(b, a);
619 }
620 
621 // Add specializations for min/max with prescribed NaN propagation.
622 template <>
623 EIGEN_STRONG_INLINE Packet16f pmin<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
624  return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
625 }
626 template <>
627 EIGEN_STRONG_INLINE Packet8d pmin<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
628  return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
629 }
630 template <>
631 EIGEN_STRONG_INLINE Packet16f pmax<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
632  return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
633 }
634 template <>
635 EIGEN_STRONG_INLINE Packet8d pmax<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
636  return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
637 }
638 template <>
639 EIGEN_STRONG_INLINE Packet16f pmin<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
640  return pminmax_propagate_nan(a, b, pmin<Packet16f>);
641 }
642 template <>
643 EIGEN_STRONG_INLINE Packet8d pmin<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
644  return pminmax_propagate_nan(a, b, pmin<Packet8d>);
645 }
646 template <>
647 EIGEN_STRONG_INLINE Packet16f pmax<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
648  return pminmax_propagate_nan(a, b, pmax<Packet16f>);
649 }
650 template <>
651 EIGEN_STRONG_INLINE Packet8d pmax<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
652  return pminmax_propagate_nan(a, b, pmax<Packet8d>);
653 }
654 
655 #ifdef EIGEN_VECTORIZE_AVX512DQ
656 template <int I_>
657 EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) {
658  return _mm512_extractf32x8_ps(x, I_);
659 }
660 template <int I_>
661 EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) {
662  return _mm512_extractf64x2_pd(x, I_);
663 }
664 EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) {
665  return _mm512_insertf32x8(_mm512_castps256_ps512(a), b, 1);
666 }
667 EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) {
668  return _mm512_inserti32x8(_mm512_castsi256_si512(a), b, 1);
669 }
670 #else
671 // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
672 template <int I_>
673 EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) {
674  return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(x), I_));
675 }
676 
677 // AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512
678 template <int I_>
679 EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) {
680  return _mm_castsi128_pd(_mm512_extracti32x4_epi32(_mm512_castpd_si512(x), I_));
681 }
682 
683 EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) {
684  return _mm512_castsi512_ps(
685  _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b), 1));
686 }
687 EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) {
688  return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1);
689 }
690 #endif
691 
692 // Helper function for bit packing snippet of low precision comparison.
693 // It packs the flags from 32x16 to 16x16.
694 EIGEN_STRONG_INLINE __m256i Pack32To16(Packet16f rf) {
695  // Split data into small pieces and handle with AVX instructions
696  // to guarantee internal order of vector.
697  // Operation:
698  // dst[15:0] := Saturate16(rf[31:0])
699  // dst[31:16] := Saturate16(rf[63:32])
700  // ...
701  // dst[255:240] := Saturate16(rf[255:224])
702  __m256i lo = _mm256_castps_si256(extract256<0>(rf));
703  __m256i hi = _mm256_castps_si256(extract256<1>(rf));
704  __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0), _mm256_extractf128_si256(lo, 1));
705  __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0), _mm256_extractf128_si256(hi, 1));
706  return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1);
707 }
708 
709 template <>
710 EIGEN_STRONG_INLINE Packet16f pisnan(const Packet16f& a) {
711  __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q);
712  return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, int32_t(-1)));
713 }
714 
715 template <>
716 EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) {
717  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
718  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
719 }
720 template <>
721 EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) {
722  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
723  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
724 }
725 
726 template <>
727 EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) {
728  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
729  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
730 }
731 
732 template <>
733 EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) {
734  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
735  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
736 }
737 
738 template <>
739 EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) {
740  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ);
741  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
742 }
743 template <>
744 EIGEN_STRONG_INLINE Packet16i pcmp_le(const Packet16i& a, const Packet16i& b) {
745  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE);
746  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
747 }
748 template <>
749 EIGEN_STRONG_INLINE Packet16i pcmp_lt(const Packet16i& a, const Packet16i& b) {
750  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT);
751  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
752 }
753 
754 template <>
755 EIGEN_STRONG_INLINE Packet8l pcmp_eq(const Packet8l& a, const Packet8l& b) {
756  __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_EQ);
757  return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
758 }
759 template <>
760 EIGEN_STRONG_INLINE Packet8l pcmp_le(const Packet8l& a, const Packet8l& b) {
761  __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LE);
762  return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
763 }
764 template <>
765 EIGEN_STRONG_INLINE Packet8l pcmp_lt(const Packet8l& a, const Packet8l& b) {
766  __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LT);
767  return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
768 }
769 
770 template <>
771 EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) {
772  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
773  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
774 }
775 template <>
776 EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) {
777  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
778  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
779 }
780 template <>
781 EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) {
782  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
783  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
784 }
785 template <>
786 EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) {
787  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
788  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
789 }
790 
791 template <>
792 EIGEN_STRONG_INLINE Packet16f print<Packet16f>(const Packet16f& a) {
793  return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION);
794 }
795 template <>
796 EIGEN_STRONG_INLINE Packet8d print<Packet8d>(const Packet8d& a) {
797  return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION);
798 }
799 
800 template <>
801 EIGEN_STRONG_INLINE Packet16f pceil<Packet16f>(const Packet16f& a) {
802  return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF);
803 }
804 template <>
805 EIGEN_STRONG_INLINE Packet8d pceil<Packet8d>(const Packet8d& a) {
806  return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF);
807 }
808 
809 template <>
810 EIGEN_STRONG_INLINE Packet16f pfloor<Packet16f>(const Packet16f& a) {
811  return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF);
812 }
813 template <>
814 EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(const Packet8d& a) {
815  return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF);
816 }
817 
818 template <>
819 EIGEN_STRONG_INLINE Packet16f ptrunc<Packet16f>(const Packet16f& a) {
820  return _mm512_roundscale_ps(a, _MM_FROUND_TO_ZERO);
821 }
822 template <>
823 EIGEN_STRONG_INLINE Packet8d ptrunc<Packet8d>(const Packet8d& a) {
824  return _mm512_roundscale_pd(a, _MM_FROUND_TO_ZERO);
825 }
826 
827 template <>
828 EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) {
829  return _mm512_set1_epi32(int32_t(-1));
830 }
831 
832 template <>
833 EIGEN_STRONG_INLINE Packet8l ptrue<Packet8l>(const Packet8l& /*a*/) {
834  return _mm512_set1_epi64(int64_t(-1));
835 }
836 
837 template <>
838 EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(const Packet16f& a) {
839  return _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
840 }
841 
842 template <>
843 EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(const Packet8d& a) {
844  return _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
845 }
846 
847 template <>
848 EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(const Packet16i& a, const Packet16i& b) {
849  return _mm512_and_si512(a, b);
850 }
851 
852 template <>
853 EIGEN_STRONG_INLINE Packet8l pand<Packet8l>(const Packet8l& a, const Packet8l& b) {
854  return _mm512_and_si512(a, b);
855 }
856 
857 template <>
858 EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a, const Packet16f& b) {
859 #ifdef EIGEN_VECTORIZE_AVX512DQ
860  return _mm512_and_ps(a, b);
861 #else
862  return _mm512_castsi512_ps(pand(_mm512_castps_si512(a), _mm512_castps_si512(b)));
863 #endif
864 }
865 template <>
866 EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a, const Packet8d& b) {
867 #ifdef EIGEN_VECTORIZE_AVX512DQ
868  return _mm512_and_pd(a, b);
869 #else
870  Packet8d res = _mm512_undefined_pd();
871  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
872  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
873  res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
874 
875  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
876  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
877  return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
878 #endif
879 }
880 
881 template <>
882 EIGEN_STRONG_INLINE Packet16i por<Packet16i>(const Packet16i& a, const Packet16i& b) {
883  return _mm512_or_si512(a, b);
884 }
885 
886 template <>
887 EIGEN_STRONG_INLINE Packet8l por<Packet8l>(const Packet8l& a, const Packet8l& b) {
888  return _mm512_or_si512(a, b);
889 }
890 
891 template <>
892 EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, const Packet16f& b) {
893 #ifdef EIGEN_VECTORIZE_AVX512DQ
894  return _mm512_or_ps(a, b);
895 #else
896  return _mm512_castsi512_ps(por(_mm512_castps_si512(a), _mm512_castps_si512(b)));
897 #endif
898 }
899 
900 template <>
901 EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a, const Packet8d& b) {
902 #ifdef EIGEN_VECTORIZE_AVX512DQ
903  return _mm512_or_pd(a, b);
904 #else
905  return _mm512_castsi512_pd(por(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
906 #endif
907 }
908 
909 template <>
910 EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(const Packet16i& a, const Packet16i& b) {
911  return _mm512_xor_si512(a, b);
912 }
913 
914 template <>
915 EIGEN_STRONG_INLINE Packet8l pxor<Packet8l>(const Packet8l& a, const Packet8l& b) {
916  return _mm512_xor_si512(a, b);
917 }
918 
919 template <>
920 EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, const Packet16f& b) {
921 #ifdef EIGEN_VECTORIZE_AVX512DQ
922  return _mm512_xor_ps(a, b);
923 #else
924  return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a), _mm512_castps_si512(b)));
925 #endif
926 }
927 
928 template <>
929 EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, const Packet8d& b) {
930 #ifdef EIGEN_VECTORIZE_AVX512DQ
931  return _mm512_xor_pd(a, b);
932 #else
933  return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
934 #endif
935 }
936 
937 template <>
938 EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(const Packet16i& a, const Packet16i& b) {
939  return _mm512_andnot_si512(b, a);
940 }
941 
942 template <>
943 EIGEN_STRONG_INLINE Packet8l pandnot<Packet8l>(const Packet8l& a, const Packet8l& b) {
944  return _mm512_andnot_si512(b, a);
945 }
946 
947 template <>
948 EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, const Packet16f& b) {
949 #ifdef EIGEN_VECTORIZE_AVX512DQ
950  return _mm512_andnot_ps(b, a);
951 #else
952  return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a), _mm512_castps_si512(b)));
953 #endif
954 }
955 template <>
956 EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a, const Packet8d& b) {
957 #ifdef EIGEN_VECTORIZE_AVX512DQ
958  return _mm512_andnot_pd(b, a);
959 #else
960  return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
961 #endif
962 }
963 
964 template <>
965 EIGEN_STRONG_INLINE Packet16f pround<Packet16f>(const Packet16f& a) {
966  // Work-around for default std::round rounding mode.
967  const Packet16f mask = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x80000000u));
968  const Packet16f prev0dot5 = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
969  return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
970 }
971 template <>
972 EIGEN_STRONG_INLINE Packet8d pround<Packet8d>(const Packet8d& a) {
973  // Work-around for default std::round rounding mode.
974  const Packet8d mask = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
975  const Packet8d prev0dot5 = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
976  return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
977 }
978 
979 template <int N>
980 EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(Packet16i a) {
981  return _mm512_srai_epi32(a, N);
982 }
983 
984 template <int N>
985 EIGEN_STRONG_INLINE Packet16i plogical_shift_right(Packet16i a) {
986  return _mm512_srli_epi32(a, N);
987 }
988 
989 template <int N>
990 EIGEN_STRONG_INLINE Packet16i plogical_shift_left(Packet16i a) {
991  return _mm512_slli_epi32(a, N);
992 }
993 
994 template <int N>
995 EIGEN_STRONG_INLINE Packet8l parithmetic_shift_right(Packet8l a) {
996  return _mm512_srai_epi64(a, N);
997 }
998 
999 template <int N>
1000 EIGEN_STRONG_INLINE Packet8l plogical_shift_right(Packet8l a) {
1001  return _mm512_srli_epi64(a, N);
1002 }
1003 
1004 template <int N>
1005 EIGEN_STRONG_INLINE Packet8l plogical_shift_left(Packet8l a) {
1006  return _mm512_slli_epi64(a, N);
1007 }
1008 
1009 template <>
1010 EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
1011  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
1012 }
1013 template <>
1014 EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
1015  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
1016 }
1017 template <>
1018 EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
1019  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
1020 }
1021 template <>
1022 EIGEN_STRONG_INLINE Packet8l pload<Packet8l>(const int64_t* from) {
1023  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
1024 }
1025 
1026 template <>
1027 EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
1028  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
1029 }
1030 template <>
1031 EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
1032  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
1033 }
1034 template <>
1035 EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
1036  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi32(from);
1037 }
1038 template <>
1039 EIGEN_STRONG_INLINE Packet8l ploadu<Packet8l>(const int64_t* from) {
1040  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi64(from);
1041 }
1042 
1043 template <>
1044 EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from, uint16_t umask) {
1045  __mmask16 mask = static_cast<__mmask16>(umask);
1046  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from);
1047 }
1048 template <>
1049 EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from, uint8_t umask) {
1050  __mmask8 mask = static_cast<__mmask8>(umask);
1051  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_pd(mask, from);
1052 }
1053 
1054 // Loads 8 floats from memory a returns the packet
1055 // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
1056 template <>
1057 EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
1058  // an unaligned load is required here as there is no requirement
1059  // on the alignment of input pointer 'from'
1060  __m256i low_half = _mm256_castps_si256(_mm256_loadu_ps(from));
1061  __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1062  __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1063  return pairs;
1064 }
1065 
1066 // Loads 4 doubles from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3,
1067 // a3}
1068 template <>
1069 EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
1070  Packet8d tmp = _mm512_castpd256_pd512(ploadu<Packet4d>(from));
1071  const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1072  return _mm512_permutexvar_pd(scatter_mask, tmp);
1073 }
1074 
1075 // Loads 4 int64_t from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3,
1076 // a3}
1077 template <>
1078 EIGEN_STRONG_INLINE Packet8l ploaddup<Packet8l>(const int64_t* from) {
1079  Packet8l tmp = _mm512_castsi256_si512(ploadu<Packet4l>(from));
1080  const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1081  return _mm512_permutexvar_epi64(scatter_mask, tmp);
1082 }
1083 
1084 // Loads 8 integers from memory and returns the packet
1085 // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
1086 template <>
1087 EIGEN_STRONG_INLINE Packet16i ploaddup<Packet16i>(const int* from) {
1088  __m256i low_half = _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1089  __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1090  __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1091  return _mm512_castps_si512(pairs);
1092 }
1093 
1094 // Loads 4 floats from memory a returns the packet
1095 // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
1096 template <>
1097 EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
1098  Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
1099  const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1100  return _mm512_permutexvar_ps(scatter_mask, tmp);
1101 }
1102 
1103 // Loads 2 doubles from memory a returns the packet
1104 // {a0, a0 a0, a0, a1, a1, a1, a1}
1105 template <>
1106 EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
1107  __m256d lane0 = _mm256_set1_pd(*from);
1108  __m256d lane1 = _mm256_set1_pd(*(from + 1));
1109  __m512d tmp = _mm512_undefined_pd();
1110  tmp = _mm512_insertf64x4(tmp, lane0, 0);
1111  return _mm512_insertf64x4(tmp, lane1, 1);
1112 }
1113 
1114 // Loads 2 int64_t from memory a returns the packet
1115 // {a0, a0 a0, a0, a1, a1, a1, a1}
1116 template <>
1117 EIGEN_STRONG_INLINE Packet8l ploadquad<Packet8l>(const int64_t* from) {
1118  __m256i lane0 = _mm256_set1_epi64x(*from);
1119  __m256i lane1 = _mm256_set1_epi64x(*(from + 1));
1120  __m512i tmp = _mm512_undefined_epi32();
1121  tmp = _mm512_inserti64x4(tmp, lane0, 0);
1122  return _mm512_inserti64x4(tmp, lane1, 1);
1123 }
1124 
1125 // Loads 4 integers from memory and returns the packet
1126 // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
1127 template <>
1128 EIGEN_STRONG_INLINE Packet16i ploadquad<Packet16i>(const int* from) {
1129  Packet16i tmp = _mm512_castsi128_si512(ploadu<Packet4i>(from));
1130  const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1131  return _mm512_permutexvar_epi32(scatter_mask, tmp);
1132 }
1133 
1134 template <>
1135 EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
1136  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
1137 }
1138 template <>
1139 EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
1140  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
1141 }
1142 template <>
1143 EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
1144  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi32(to, from);
1145 }
1146 template <>
1147 EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet8l& from) {
1148  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi64(to, from);
1149 }
1150 
1151 template <>
1152 EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
1153  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
1154 }
1155 template <>
1156 EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
1157  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
1158 }
1159 template <>
1160 EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
1161  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi32(to, from);
1162 }
1163 template <>
1164 EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet8l& from) {
1165  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi64(to, from);
1166 }
1167 template <>
1168 EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from, uint16_t umask) {
1169  __mmask16 mask = static_cast<__mmask16>(umask);
1170  EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from);
1171 }
1172 template <>
1173 EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from, uint8_t umask) {
1174  __mmask8 mask = static_cast<__mmask8>(umask);
1175  EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from);
1176 }
1177 
1178 template <typename Scalar, typename Packet>
1179 EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from, Index stride,
1180  typename unpacket_traits<Packet>::mask_t umask);
1181 template <>
1182 EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const Packet16f& src, const float* from, Index stride,
1183  uint16_t umask) {
1184  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1185  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1186  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1187  __mmask16 mask = static_cast<__mmask16>(umask);
1188 
1189  return _mm512_mask_i32gather_ps(src, mask, indices, from, 4);
1190 }
1191 template <>
1192 EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const Packet8d& src, const double* from, Index stride,
1193  uint8_t umask) {
1194  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1195  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1196  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1197  __mmask8 mask = static_cast<__mmask8>(umask);
1198 
1199  return _mm512_mask_i32gather_pd(src, mask, indices, from, 8);
1200 }
1201 
1202 template <>
1203 EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from, Index stride) {
1204  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1205  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1206  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1207 
1208  return _mm512_i32gather_ps(indices, from, 4);
1209 }
1210 template <>
1211 EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from, Index stride) {
1212  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1213  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1214  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1215 
1216  return _mm512_i32gather_pd(indices, from, 8);
1217 }
1218 template <>
1219 EIGEN_DEVICE_FUNC inline Packet8l pgather<int64_t, Packet8l>(const int64_t* from, Index stride) {
1220  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1221  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1222  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1223 
1224  return _mm512_i32gather_epi64(indices, from, 8);
1225 }
1226 template <>
1227 EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from, Index stride) {
1228  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1229  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1230  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1231  return _mm512_i32gather_epi32(indices, from, 4);
1232 }
1233 
1234 template <typename Scalar, typename Packet>
1235 EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index stride,
1236  typename unpacket_traits<Packet>::mask_t umask);
1237 template <>
1238 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride,
1239  uint16_t umask) {
1240  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1241  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1242  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1243  __mmask16 mask = static_cast<__mmask16>(umask);
1244  _mm512_mask_i32scatter_ps(to, mask, indices, from, 4);
1245 }
1246 template <>
1247 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride,
1248  uint8_t umask) {
1249  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1250  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1251  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1252  __mmask8 mask = static_cast<__mmask8>(umask);
1253  _mm512_mask_i32scatter_pd(to, mask, indices, from, 8);
1254 }
1255 template <>
1256 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride) {
1257  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1258  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1259  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1260  _mm512_i32scatter_ps(to, indices, from, 4);
1261 }
1262 template <>
1263 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride) {
1264  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1265  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1266  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1267  _mm512_i32scatter_pd(to, indices, from, 8);
1268 }
1269 template <>
1270 EIGEN_DEVICE_FUNC inline void pscatter<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index stride) {
1271  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1272  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1273  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1274  _mm512_i32scatter_epi64(to, indices, from, 8);
1275 }
1276 template <>
1277 EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to, const Packet16i& from, Index stride) {
1278  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1279  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1280  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1281  _mm512_i32scatter_epi32(to, indices, from, 4);
1282 }
1283 
1284 template <>
1285 EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
1286  Packet16f pa = pset1<Packet16f>(a);
1287  pstore(to, pa);
1288 }
1289 template <>
1290 EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
1291  Packet8d pa = pset1<Packet8d>(a);
1292  pstore(to, pa);
1293 }
1294 template <>
1295 EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
1296  Packet16i pa = pset1<Packet16i>(a);
1297  pstore(to, pa);
1298 }
1299 template <>
1300 EIGEN_STRONG_INLINE void pstore1<Packet8l>(int64_t* to, const int64_t& a) {
1301  Packet8l pa = pset1<Packet8l>(a);
1302  pstore(to, pa);
1303 }
1304 
1305 template <>
1306 EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1307  _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1308 }
1309 template <>
1310 EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1311  _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1312 }
1313 template <>
1314 EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
1315  _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1316 }
1317 
1318 template <>
1319 EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
1320  return _mm512_cvtss_f32(a);
1321 }
1322 template <>
1323 EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
1324  return _mm512_cvtsd_f64(a);
1325 }
1326 template <>
1327 EIGEN_STRONG_INLINE int64_t pfirst<Packet8l>(const Packet8l& a) {
1328  int64_t x = _mm_extract_epi64_0(_mm512_extracti32x4_epi32(a, 0));
1329  return x;
1330 }
1331 template <>
1332 EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
1333 #if EIGEN_GNUC_STRICT_LESS_THAN(11, 0, 0)
1334  return _mm_cvtsi128_si32(_mm512_castsi512_si128(a));
1335 #else
1336  return _mm512_cvtsi512_si32(a);
1337 #endif
1338 }
1339 
1340 template <>
1341 EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) {
1342  return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1343 }
1344 
1345 template <>
1346 EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) {
1347  return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
1348 }
1349 
1350 template <>
1351 EIGEN_STRONG_INLINE Packet16i preverse(const Packet16i& a) {
1352  return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1353 }
1354 
1355 template <>
1356 EIGEN_STRONG_INLINE Packet8l preverse(const Packet8l& a) {
1357  return _mm512_permutexvar_epi64(_mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7), a);
1358 }
1359 
1360 template <>
1361 EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) {
1362  // _mm512_abs_ps intrinsic not found, so hack around it
1363  return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
1364 }
1365 template <>
1366 EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
1367  // _mm512_abs_ps intrinsic not found, so hack around it
1368  return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff)));
1369 }
1370 template <>
1371 EIGEN_STRONG_INLINE Packet16i pabs(const Packet16i& a) {
1372  return _mm512_abs_epi32(a);
1373 }
1374 template <>
1375 EIGEN_STRONG_INLINE Packet8l pabs(const Packet8l& a) {
1376  return _mm512_abs_epi64(a);
1377 }
1378 
1379 #ifndef EIGEN_VECTORIZE_AVX512FP16
1380 template <>
1381 EIGEN_STRONG_INLINE Packet16h psignbit(const Packet16h& a) {
1382  return _mm256_srai_epi16(a, 15);
1383 }
1384 #endif // EIGEN_VECTORIZE_AVX512FP16
1385 
1386 template <>
1387 EIGEN_STRONG_INLINE Packet16bf psignbit(const Packet16bf& a) {
1388  return _mm256_srai_epi16(a, 15);
1389 }
1390 template <>
1391 EIGEN_STRONG_INLINE Packet16f psignbit(const Packet16f& a) {
1392  return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31));
1393 }
1394 template <>
1395 EIGEN_STRONG_INLINE Packet8d psignbit(const Packet8d& a) {
1396  return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63));
1397 }
1398 
1399 template <>
1400 EIGEN_STRONG_INLINE Packet16f pfrexp<Packet16f>(const Packet16f& a, Packet16f& exponent) {
1401  return pfrexp_generic(a, exponent);
1402 }
1403 
1404 // Extract exponent without existence of Packet8l.
1405 template <>
1406 EIGEN_STRONG_INLINE Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) {
1407  const Packet8d cst_exp_mask = pset1frombits<Packet8d>(static_cast<uint64_t>(0x7ff0000000000000ull));
1408 #ifdef EIGEN_VECTORIZE_AVX512DQ
1409  return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
1410 #else
1411  return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
1412 #endif
1413 }
1414 
1415 template <>
1416 EIGEN_STRONG_INLINE Packet8d pfrexp<Packet8d>(const Packet8d& a, Packet8d& exponent) {
1417  return pfrexp_generic(a, exponent);
1418 }
1419 
1420 template <>
1421 EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(const Packet16f& a, const Packet16f& exponent) {
1422  return pldexp_generic(a, exponent);
1423 }
1424 
1425 template <>
1426 EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(const Packet8d& a, const Packet8d& exponent) {
1427  // Clamp exponent to [-2099, 2099]
1428  const Packet8d max_exponent = pset1<Packet8d>(2099.0);
1429  const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1430 
1431  // Split 2^e into four factors and multiply.
1432  const Packet8i bias = pset1<Packet8i>(1023);
1433  Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4)
1434 
1435  // 2^b
1436  const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
1437  Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
1438  Packet8i lo = _mm256_slli_epi64(hi, 52);
1439  hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
1440  Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
1441  Packet8d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b)
1442 
1443  // 2^(e - 3b)
1444  b = psub(psub(psub(e, b), b), b); // e - 3b
1445  hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
1446  lo = _mm256_slli_epi64(hi, 52);
1447  hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
1448  c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
1449  out = pmul(out, c); // a * 2^e
1450  return out;
1451 }
1452 
1453 #ifdef EIGEN_VECTORIZE_AVX512DQ
1454 // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
1455 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1456  __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \
1457  __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1)
1458 
1459 // AVX512F does not define _mm512_extracti32x8_epi32 to extract _m256i from _m512i
1460 #define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1461  __m256i OUTPUT##_0 = _mm512_extracti32x8_epi32(INPUT, 0); \
1462  __m256i OUTPUT##_1 = _mm512_extracti32x8_epi32(INPUT, 1)
1463 #else
1464 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1465  __m256 OUTPUT##_0 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
1466  _mm512_extractf32x4_ps(INPUT, 1), 1); \
1467  __m256 OUTPUT##_1 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
1468  _mm512_extractf32x4_ps(INPUT, 3), 1)
1469 
1470 #define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1471  __m256i OUTPUT##_0 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 0)), \
1472  _mm512_extracti32x4_epi32(INPUT, 1), 1); \
1473  __m256i OUTPUT##_1 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 2)), \
1474  _mm512_extracti32x4_epi32(INPUT, 3), 1)
1475 #endif
1476 
1477 #ifdef EIGEN_VECTORIZE_AVX512DQ
1478 #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1479  OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1);
1480 
1481 #define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1482  OUTPUT = _mm512_inserti32x8(_mm512_castsi256_si512(INPUTA), INPUTB, 1);
1483 #else
1484 #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1485  OUTPUT = _mm512_undefined_ps(); \
1486  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
1487  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
1488  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
1489  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
1490 
1491 #define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1492  OUTPUT = _mm512_undefined_epi32(); \
1493  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 0), 0); \
1494  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 1), 1); \
1495  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 0), 2); \
1496  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 1), 3);
1497 #endif
1498 
1499 template <>
1500 EIGEN_STRONG_INLINE Packet8f predux_half_dowto4<Packet16f>(const Packet16f& a) {
1501 #ifdef EIGEN_VECTORIZE_AVX512DQ
1502  __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1503  __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1504  return _mm256_add_ps(lane0, lane1);
1505 #else
1506  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1507  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1508  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1509  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1510  __m128 sum0 = _mm_add_ps(lane0, lane2);
1511  __m128 sum1 = _mm_add_ps(lane1, lane3);
1512  return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
1513 #endif
1514 }
1515 template <>
1516 EIGEN_STRONG_INLINE Packet4d predux_half_dowto4<Packet8d>(const Packet8d& a) {
1517  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1518  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1519  return _mm256_add_pd(lane0, lane1);
1520 }
1521 template <>
1522 EIGEN_STRONG_INLINE Packet8i predux_half_dowto4<Packet16i>(const Packet16i& a) {
1523 #ifdef EIGEN_VECTORIZE_AVX512DQ
1524  __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
1525  __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
1526  return _mm256_add_epi32(lane0, lane1);
1527 #else
1528  __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
1529  __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
1530  __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
1531  __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
1532  __m128i sum0 = _mm_add_epi32(lane0, lane2);
1533  __m128i sum1 = _mm_add_epi32(lane1, lane3);
1534  return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1);
1535 #endif
1536 }
1537 
1538 template <>
1539 EIGEN_STRONG_INLINE Packet4l predux_half_dowto4<Packet8l>(const Packet8l& a) {
1540  __m256i lane0 = _mm512_extracti64x4_epi64(a, 0);
1541  __m256i lane1 = _mm512_extracti64x4_epi64(a, 1);
1542  return _mm256_add_epi64(lane0, lane1);
1543 }
1544 
1545 #define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1546  EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1547 
1548 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1549  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1550  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1551  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1552  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1553  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1554  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1555  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1556  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1557  __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1558  __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1559  __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1560  __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1561  __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1562  __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1563  __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1564  __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1565  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1566  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1567  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1568  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1569  __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1570  __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1571  __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1572  __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1573  __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1574  __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1575  __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1576  __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1577  __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1578  __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1579  __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1580  __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1581 
1582  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1583  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1584  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1585  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1586  EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1587  EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1588  EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1589  EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1590  EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1591  EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1592  EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1593  EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1594  EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1595  EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1596  EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1597  EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1598 
1599  PacketBlock<Packet8f, 32> tmp;
1600 
1601  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1602  tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1603  tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1604  tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1605  tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1606  tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1607  tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1608  tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1609 
1610  tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1611  tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1612  tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1613  tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1614  tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1615  tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1616  tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1617  tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1618 
1619  // Second set of _m256 outputs
1620  tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1621  tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1622  tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1623  tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1624  tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1625  tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1626  tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1627  tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1628 
1629  tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1630  tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1631  tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1632  tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1633  tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1634  tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1635  tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1636  tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1637 
1638  // Pack them into the output
1639  PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1640  PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1641  PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1642  PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1643 
1644  PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1645  PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1646  PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1647  PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1648 
1649  PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1650  PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1651  PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1652  PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1653 
1654  PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1655  PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1656  PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1657  PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1658 }
1659 #define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1660  EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1661 
1662 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 8>& kernel) {
1663  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1664  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1665  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1666  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1667  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1668  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1669  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1670  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1671 
1672  kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1673  kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1674  kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1675  kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1676  kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1677  kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1678  kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1679  kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1680 
1681  T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44);
1682  T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee);
1683  T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44);
1684  T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee);
1685  T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44);
1686  T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee);
1687  T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44);
1688  T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee);
1689 
1690  kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88);
1691  kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd);
1692  kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88);
1693  kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd);
1694  kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88);
1695  kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd);
1696  kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88);
1697  kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd);
1698 }
1699 
1700 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1701  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1702  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1703  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1704  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1705 
1706  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1707  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1708  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1709  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1710 
1711  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1712  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1713  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1714  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1715 
1716  PacketBlock<Packet8f, 8> tmp;
1717 
1718  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1719  tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1720  tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1721  tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1722 
1723  tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1724  tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1725  tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1726  tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1727 
1728  PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1729  PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1730  PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1731  PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1732 }
1733 
1734 #define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1735  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1736  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1737 
1738 #define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1739  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1740  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1741 
1742 #define PACK_OUTPUT_L(OUTPUT, INPUT, INDEX, STRIDE) \
1743  OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1744  OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1745 
1746 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1747  __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1748  __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1749  __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1750  __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1751 
1752  PacketBlock<Packet4d, 8> tmp;
1753 
1754  tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20);
1755  tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20);
1756  tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31);
1757  tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31);
1758 
1759  tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20);
1760  tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20);
1761  tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31);
1762  tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31);
1763 
1764  PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1765  PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1766  PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1767  PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1768 }
1769 
1770 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1771  __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1772  __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1773  __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1774  __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1775  __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1776  __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1777  __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1778  __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1779 
1780  kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E);
1781  kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]);
1782  kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E);
1783  kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2);
1784  kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E);
1785  kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]);
1786  kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E);
1787  kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3);
1788  kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E);
1789  kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]);
1790  kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E);
1791  kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6);
1792  kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E);
1793  kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]);
1794  kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E);
1795  kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7);
1796 
1797  T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1798  T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0);
1799  T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1800  T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]);
1801  T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1802  T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1);
1803  T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1804  T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]);
1805  T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1806  T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2);
1807  T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1808  T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]);
1809  T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1810  T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3);
1811  T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1812  T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]);
1813 
1814  kernel.packet[0] = T0;
1815  kernel.packet[1] = T1;
1816  kernel.packet[2] = T2;
1817  kernel.packet[3] = T3;
1818  kernel.packet[4] = T4;
1819  kernel.packet[5] = T5;
1820  kernel.packet[6] = T6;
1821  kernel.packet[7] = T7;
1822 }
1823 
1824 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8l, 4>& kernel) {
1825  __m512i T0 = _mm512_castpd_si512(
1826  _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0));
1827  __m512i T1 = _mm512_castpd_si512(
1828  _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0xff));
1829  __m512i T2 = _mm512_castpd_si512(
1830  _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0));
1831  __m512i T3 = _mm512_castpd_si512(
1832  _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0xff));
1833 
1834  PacketBlock<Packet4l, 8> tmp;
1835 
1836  tmp.packet[0] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x20);
1837  tmp.packet[1] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x20);
1838  tmp.packet[2] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x31);
1839  tmp.packet[3] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x31);
1840 
1841  tmp.packet[4] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x20);
1842  tmp.packet[5] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x20);
1843  tmp.packet[6] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x31);
1844  tmp.packet[7] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x31);
1845 
1846  PACK_OUTPUT_L(kernel.packet, tmp.packet, 0, 1);
1847  PACK_OUTPUT_L(kernel.packet, tmp.packet, 1, 1);
1848  PACK_OUTPUT_L(kernel.packet, tmp.packet, 2, 1);
1849  PACK_OUTPUT_L(kernel.packet, tmp.packet, 3, 1);
1850 }
1851 
1852 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8l, 8>& kernel) {
1853  __m512i T0 = _mm512_unpacklo_epi64(kernel.packet[0], kernel.packet[1]);
1854  __m512i T1 = _mm512_unpackhi_epi64(kernel.packet[0], kernel.packet[1]);
1855  __m512i T2 = _mm512_unpacklo_epi64(kernel.packet[2], kernel.packet[3]);
1856  __m512i T3 = _mm512_unpackhi_epi64(kernel.packet[2], kernel.packet[3]);
1857  __m512i T4 = _mm512_unpacklo_epi64(kernel.packet[4], kernel.packet[5]);
1858  __m512i T5 = _mm512_unpackhi_epi64(kernel.packet[4], kernel.packet[5]);
1859  __m512i T6 = _mm512_unpacklo_epi64(kernel.packet[6], kernel.packet[7]);
1860  __m512i T7 = _mm512_unpackhi_epi64(kernel.packet[6], kernel.packet[7]);
1861 
1862  kernel.packet[0] = _mm512_permutex_epi64(T2, 0x4E);
1863  kernel.packet[0] = _mm512_mask_blend_epi64(0xCC, T0, kernel.packet[0]);
1864  kernel.packet[2] = _mm512_permutex_epi64(T0, 0x4E);
1865  kernel.packet[2] = _mm512_mask_blend_epi64(0xCC, kernel.packet[2], T2);
1866  kernel.packet[1] = _mm512_permutex_epi64(T3, 0x4E);
1867  kernel.packet[1] = _mm512_mask_blend_epi64(0xCC, T1, kernel.packet[1]);
1868  kernel.packet[3] = _mm512_permutex_epi64(T1, 0x4E);
1869  kernel.packet[3] = _mm512_mask_blend_epi64(0xCC, kernel.packet[3], T3);
1870  kernel.packet[4] = _mm512_permutex_epi64(T6, 0x4E);
1871  kernel.packet[4] = _mm512_mask_blend_epi64(0xCC, T4, kernel.packet[4]);
1872  kernel.packet[6] = _mm512_permutex_epi64(T4, 0x4E);
1873  kernel.packet[6] = _mm512_mask_blend_epi64(0xCC, kernel.packet[6], T6);
1874  kernel.packet[5] = _mm512_permutex_epi64(T7, 0x4E);
1875  kernel.packet[5] = _mm512_mask_blend_epi64(0xCC, T5, kernel.packet[5]);
1876  kernel.packet[7] = _mm512_permutex_epi64(T5, 0x4E);
1877  kernel.packet[7] = _mm512_mask_blend_epi64(0xCC, kernel.packet[7], T7);
1878 
1879  T0 = _mm512_shuffle_i64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1880  T0 = _mm512_mask_blend_epi64(0xF0, kernel.packet[0], T0);
1881  T4 = _mm512_shuffle_i64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1882  T4 = _mm512_mask_blend_epi64(0xF0, T4, kernel.packet[4]);
1883  T1 = _mm512_shuffle_i64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1884  T1 = _mm512_mask_blend_epi64(0xF0, kernel.packet[1], T1);
1885  T5 = _mm512_shuffle_i64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1886  T5 = _mm512_mask_blend_epi64(0xF0, T5, kernel.packet[5]);
1887  T2 = _mm512_shuffle_i64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1888  T2 = _mm512_mask_blend_epi64(0xF0, kernel.packet[2], T2);
1889  T6 = _mm512_shuffle_i64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1890  T6 = _mm512_mask_blend_epi64(0xF0, T6, kernel.packet[6]);
1891  T3 = _mm512_shuffle_i64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1892  T3 = _mm512_mask_blend_epi64(0xF0, kernel.packet[3], T3);
1893  T7 = _mm512_shuffle_i64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1894  T7 = _mm512_mask_blend_epi64(0xF0, T7, kernel.packet[7]);
1895 
1896  kernel.packet[0] = T0;
1897  kernel.packet[1] = T1;
1898  kernel.packet[2] = T2;
1899  kernel.packet[3] = T3;
1900  kernel.packet[4] = T4;
1901  kernel.packet[5] = T5;
1902  kernel.packet[6] = T6;
1903  kernel.packet[7] = T7;
1904 }
1905 
1906 #define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE) \
1907  EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1908 
1909 #define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE) \
1910  EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1911 
1912 #define SHUFFLE_EPI32(A, B, M) _mm512_castps_si512(_mm512_shuffle_ps(_mm512_castsi512_ps(A), _mm512_castsi512_ps(B), M))
1913 
1914 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 16>& kernel) {
1915  __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1916  __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1917  __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1918  __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1919  __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]);
1920  __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]);
1921  __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]);
1922  __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]);
1923  __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]);
1924  __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]);
1925  __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]);
1926  __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]);
1927  __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]);
1928  __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]);
1929  __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]);
1930  __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]);
1931  __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1932  __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1933  __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1934  __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1935  __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1936  __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1937  __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1938  __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1939  __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1940  __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1941  __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1942  __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1943  __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1944  __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1945  __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1946  __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1947 
1948  EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
1949  EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
1950  EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
1951  EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
1952  EIGEN_EXTRACT_8i_FROM_16i(S4, S4);
1953  EIGEN_EXTRACT_8i_FROM_16i(S5, S5);
1954  EIGEN_EXTRACT_8i_FROM_16i(S6, S6);
1955  EIGEN_EXTRACT_8i_FROM_16i(S7, S7);
1956  EIGEN_EXTRACT_8i_FROM_16i(S8, S8);
1957  EIGEN_EXTRACT_8i_FROM_16i(S9, S9);
1958  EIGEN_EXTRACT_8i_FROM_16i(S10, S10);
1959  EIGEN_EXTRACT_8i_FROM_16i(S11, S11);
1960  EIGEN_EXTRACT_8i_FROM_16i(S12, S12);
1961  EIGEN_EXTRACT_8i_FROM_16i(S13, S13);
1962  EIGEN_EXTRACT_8i_FROM_16i(S14, S14);
1963  EIGEN_EXTRACT_8i_FROM_16i(S15, S15);
1964 
1965  PacketBlock<Packet8i, 32> tmp;
1966 
1967  tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20);
1968  tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20);
1969  tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20);
1970  tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20);
1971  tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31);
1972  tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31);
1973  tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31);
1974  tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31);
1975 
1976  tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20);
1977  tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20);
1978  tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20);
1979  tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20);
1980  tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31);
1981  tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31);
1982  tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31);
1983  tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31);
1984 
1985  // Second set of _m256 outputs
1986  tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20);
1987  tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20);
1988  tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20);
1989  tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20);
1990  tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31);
1991  tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31);
1992  tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31);
1993  tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31);
1994 
1995  tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20);
1996  tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20);
1997  tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20);
1998  tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20);
1999  tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31);
2000  tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31);
2001  tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31);
2002  tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31);
2003 
2004  // Pack them into the output
2005  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16);
2006  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16);
2007  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16);
2008  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16);
2009 
2010  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16);
2011  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16);
2012  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16);
2013  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16);
2014 
2015  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16);
2016  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16);
2017  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16);
2018  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16);
2019 
2020  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16);
2021  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16);
2022  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16);
2023  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16);
2024 }
2025 
2026 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 4>& kernel) {
2027  __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
2028  __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
2029  __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
2030  __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
2031 
2032  __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2033  __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2034  __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2035  __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2036 
2037  EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
2038  EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
2039  EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
2040  EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
2041 
2042  PacketBlock<Packet8i, 8> tmp;
2043 
2044  tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20);
2045  tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20);
2046  tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31);
2047  tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31);
2048 
2049  tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20);
2050  tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20);
2051  tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31);
2052  tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31);
2053 
2054  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1);
2055  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1);
2056  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1);
2057  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1);
2058 }
2059 
2060 template <size_t N>
2061 EIGEN_STRONG_INLINE int avx512_blend_mask(const Selector<N>& ifPacket) {
2062  alignas(__m128i) uint8_t aux[sizeof(__m128i)];
2063  for (size_t i = 0; i < N; i++) aux[i] = static_cast<uint8_t>(ifPacket.select[i]);
2064  __m128i paux = _mm_sub_epi8(_mm_setzero_si128(), _mm_load_si128(reinterpret_cast<const __m128i*>(aux)));
2065  return _mm_movemask_epi8(paux);
2066 }
2067 
2068 template <>
2069 EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& ifPacket, const Packet16f& thenPacket,
2070  const Packet16f& elsePacket) {
2071  __mmask16 m = avx512_blend_mask(ifPacket);
2072  return _mm512_mask_blend_ps(m, elsePacket, thenPacket);
2073 }
2074 template <>
2075 EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket, const Packet8d& thenPacket,
2076  const Packet8d& elsePacket) {
2077  __mmask8 m = avx512_blend_mask(ifPacket);
2078  return _mm512_mask_blend_pd(m, elsePacket, thenPacket);
2079 }
2080 
2081 // Packet math for Eigen::half
2082 #ifndef EIGEN_VECTORIZE_AVX512FP16
2083 template <>
2084 EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) {
2085  return _mm256_set1_epi16(from.x);
2086 }
2087 
2088 template <>
2089 EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) {
2090  return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
2091 }
2092 
2093 template <>
2094 EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) {
2095  return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2096 }
2097 
2098 template <>
2099 EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) {
2100  return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2101 }
2102 
2103 template <>
2104 EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) {
2105  // (void*) -> workaround clang warning:
2106  // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
2107  EIGEN_DEBUG_ALIGNED_STORE
2108  _mm256_store_si256((__m256i*)(void*)to, from);
2109 }
2110 
2111 template <>
2112 EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) {
2113  // (void*) -> workaround clang warning:
2114  // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
2115  EIGEN_DEBUG_UNALIGNED_STORE
2116  _mm256_storeu_si256((__m256i*)(void*)to, from);
2117 }
2118 
2119 template <>
2120 EIGEN_STRONG_INLINE Packet16h ploaddup<Packet16h>(const Eigen::half* from) {
2121  unsigned short a = from[0].x;
2122  unsigned short b = from[1].x;
2123  unsigned short c = from[2].x;
2124  unsigned short d = from[3].x;
2125  unsigned short e = from[4].x;
2126  unsigned short f = from[5].x;
2127  unsigned short g = from[6].x;
2128  unsigned short h = from[7].x;
2129  return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2130 }
2131 
2132 template <>
2133 EIGEN_STRONG_INLINE Packet16h ploadquad(const Eigen::half* from) {
2134  unsigned short a = from[0].x;
2135  unsigned short b = from[1].x;
2136  unsigned short c = from[2].x;
2137  unsigned short d = from[3].x;
2138  return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2139 }
2140 
2141 EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { return _mm512_cvtph_ps(a); }
2142 
2143 EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) {
2144  return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
2145 }
2146 
2147 template <>
2148 EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) {
2149  return Packet16h(ptrue(Packet8i(a)));
2150 }
2151 
2152 template <>
2153 EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) {
2154  const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
2155  return _mm256_andnot_si256(sign_mask, a);
2156 }
2157 
2158 template <>
2159 EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a, const Packet16h& b) {
2160  return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
2161 }
2162 
2163 template <>
2164 EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a, const Packet16h& b) {
2165  return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
2166 }
2167 
2168 template <>
2169 EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) {
2170  return float2half(plset<Packet16f>(static_cast<float>(a)));
2171 }
2172 
2173 template <>
2174 EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a, const Packet16h& b) {
2175  // in some cases Packet8i is a wrapper around __m256i, so we need to
2176  // cast to Packet8i to call the correct overload.
2177  return Packet16h(por(Packet8i(a), Packet8i(b)));
2178 }
2179 template <>
2180 EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a, const Packet16h& b) {
2181  return Packet16h(pxor(Packet8i(a), Packet8i(b)));
2182 }
2183 template <>
2184 EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a, const Packet16h& b) {
2185  return Packet16h(pand(Packet8i(a), Packet8i(b)));
2186 }
2187 template <>
2188 EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a, const Packet16h& b) {
2189  return Packet16h(pandnot(Packet8i(a), Packet8i(b)));
2190 }
2191 
2192 template <>
2193 EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) {
2194  return _mm256_blendv_epi8(b, a, mask);
2195 }
2196 
2197 template <>
2198 EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) {
2199  return float2half(pround<Packet16f>(half2float(a)));
2200 }
2201 
2202 template <>
2203 EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) {
2204  return float2half(print<Packet16f>(half2float(a)));
2205 }
2206 
2207 template <>
2208 EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) {
2209  return float2half(pceil<Packet16f>(half2float(a)));
2210 }
2211 
2212 template <>
2213 EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) {
2214  return float2half(pfloor<Packet16f>(half2float(a)));
2215 }
2216 
2217 template <>
2218 EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(const Packet16h& a) {
2219  return float2half(ptrunc<Packet16f>(half2float(a)));
2220 }
2221 
2222 template <>
2223 EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) {
2224  Packet16f af = half2float(a);
2225  Packet16f bf = half2float(b);
2226  return Pack32To16(pcmp_eq(af, bf));
2227 }
2228 
2229 template <>
2230 EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a, const Packet16h& b) {
2231  return Pack32To16(pcmp_le(half2float(a), half2float(b)));
2232 }
2233 
2234 template <>
2235 EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a, const Packet16h& b) {
2236  return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
2237 }
2238 
2239 template <>
2240 EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a, const Packet16h& b) {
2241  return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
2242 }
2243 
2244 template <>
2245 EIGEN_STRONG_INLINE Packet16h pconj(const Packet16h& a) {
2246  return a;
2247 }
2248 
2249 template <>
2250 EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) {
2251  Packet16h sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2252  return _mm256_xor_si256(a, sign_mask);
2253 }
2254 
2255 template <>
2256 EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
2257  Packet16f af = half2float(a);
2258  Packet16f bf = half2float(b);
2259  Packet16f rf = padd(af, bf);
2260  return float2half(rf);
2261 }
2262 
2263 template <>
2264 EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) {
2265  Packet16f af = half2float(a);
2266  Packet16f bf = half2float(b);
2267  Packet16f rf = psub(af, bf);
2268  return float2half(rf);
2269 }
2270 
2271 template <>
2272 EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
2273  Packet16f af = half2float(a);
2274  Packet16f bf = half2float(b);
2275  Packet16f rf = pmul(af, bf);
2276  return float2half(rf);
2277 }
2278 
2279 template <>
2280 EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) {
2281  Packet16f af = half2float(a);
2282  Packet16f bf = half2float(b);
2283  Packet16f rf = pdiv(af, bf);
2284  return float2half(rf);
2285 }
2286 
2287 template <>
2288 EIGEN_STRONG_INLINE Packet16h pmadd<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2289  return float2half(pmadd(half2float(a), half2float(b), half2float(c)));
2290 }
2291 
2292 template <>
2293 EIGEN_STRONG_INLINE Packet16h pmsub<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2294  return float2half(pmsub(half2float(a), half2float(b), half2float(c)));
2295 }
2296 
2297 template <>
2298 EIGEN_STRONG_INLINE Packet16h pnmadd<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2299  return float2half(pnmadd(half2float(a), half2float(b), half2float(c)));
2300 }
2301 
2302 template <>
2303 EIGEN_STRONG_INLINE Packet16h pnmsub<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2304  return float2half(pnmsub(half2float(a), half2float(b), half2float(c)));
2305 }
2306 
2307 template <>
2308 EIGEN_STRONG_INLINE Packet8h predux_half_dowto4<Packet16h>(const Packet16h& a) {
2309  Packet8h lane0 = _mm256_extractf128_si256(a, 0);
2310  Packet8h lane1 = _mm256_extractf128_si256(a, 1);
2311  return padd<Packet8h>(lane0, lane1);
2312 }
2313 
2314 template <>
2315 EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) {
2316  __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
2317  return _mm256_insertf128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a, 1), m)),
2318  _mm_shuffle_epi8(_mm256_extractf128_si256(a, 0), m), 1);
2319 }
2320 
2321 template <>
2322 EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) {
2323  return _mm256_set_epi16(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
2324  from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x,
2325  from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x,
2326  from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x);
2327 }
2328 
2329 template <>
2330 EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) {
2331  EIGEN_ALIGN64 half aux[16];
2332  pstore(aux, from);
2333  to[stride * 0] = aux[0];
2334  to[stride * 1] = aux[1];
2335  to[stride * 2] = aux[2];
2336  to[stride * 3] = aux[3];
2337  to[stride * 4] = aux[4];
2338  to[stride * 5] = aux[5];
2339  to[stride * 6] = aux[6];
2340  to[stride * 7] = aux[7];
2341  to[stride * 8] = aux[8];
2342  to[stride * 9] = aux[9];
2343  to[stride * 10] = aux[10];
2344  to[stride * 11] = aux[11];
2345  to[stride * 12] = aux[12];
2346  to[stride * 13] = aux[13];
2347  to[stride * 14] = aux[14];
2348  to[stride * 15] = aux[15];
2349 }
2350 
2351 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 16>& kernel) {
2352  __m256i a = kernel.packet[0];
2353  __m256i b = kernel.packet[1];
2354  __m256i c = kernel.packet[2];
2355  __m256i d = kernel.packet[3];
2356  __m256i e = kernel.packet[4];
2357  __m256i f = kernel.packet[5];
2358  __m256i g = kernel.packet[6];
2359  __m256i h = kernel.packet[7];
2360  __m256i i = kernel.packet[8];
2361  __m256i j = kernel.packet[9];
2362  __m256i k = kernel.packet[10];
2363  __m256i l = kernel.packet[11];
2364  __m256i m = kernel.packet[12];
2365  __m256i n = kernel.packet[13];
2366  __m256i o = kernel.packet[14];
2367  __m256i p = kernel.packet[15];
2368 
2369  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2370  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2371  __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2372  __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2373  __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2374  __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2375  __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2376  __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2377 
2378  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2379  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2380  __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2381  __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2382  __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2383  __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2384  __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2385  __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2386 
2387  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2388  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2389  __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2390  __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2391  __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2392  __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2393  __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2394  __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2395 
2396  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2397  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2398  __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2399  __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2400  __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2401  __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2402  __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2403  __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2404 
2405  __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2406  __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2407  __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2408  __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2409  __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2410  __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2411  __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2412  __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2413  __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2414  __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2415  __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2416  __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2417  __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2418  __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2419  __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2420  __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2421 
2422  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2423  __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2424  __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2425  __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2426  __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2427  __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2428  __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2429  __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2430  __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2431  __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2432  __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2433  __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2434  __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2435  __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2436  __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2437  __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2438  __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2439 
2440  kernel.packet[0] = a_p_0;
2441  kernel.packet[1] = a_p_1;
2442  kernel.packet[2] = a_p_2;
2443  kernel.packet[3] = a_p_3;
2444  kernel.packet[4] = a_p_4;
2445  kernel.packet[5] = a_p_5;
2446  kernel.packet[6] = a_p_6;
2447  kernel.packet[7] = a_p_7;
2448  kernel.packet[8] = a_p_8;
2449  kernel.packet[9] = a_p_9;
2450  kernel.packet[10] = a_p_a;
2451  kernel.packet[11] = a_p_b;
2452  kernel.packet[12] = a_p_c;
2453  kernel.packet[13] = a_p_d;
2454  kernel.packet[14] = a_p_e;
2455  kernel.packet[15] = a_p_f;
2456 }
2457 
2458 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 8>& kernel) {
2459  EIGEN_ALIGN64 half in[8][16];
2460  pstore<half>(in[0], kernel.packet[0]);
2461  pstore<half>(in[1], kernel.packet[1]);
2462  pstore<half>(in[2], kernel.packet[2]);
2463  pstore<half>(in[3], kernel.packet[3]);
2464  pstore<half>(in[4], kernel.packet[4]);
2465  pstore<half>(in[5], kernel.packet[5]);
2466  pstore<half>(in[6], kernel.packet[6]);
2467  pstore<half>(in[7], kernel.packet[7]);
2468 
2469  EIGEN_ALIGN64 half out[8][16];
2470 
2471  for (int i = 0; i < 8; ++i) {
2472  for (int j = 0; j < 8; ++j) {
2473  out[i][j] = in[j][2 * i];
2474  }
2475  for (int j = 0; j < 8; ++j) {
2476  out[i][j + 8] = in[j][2 * i + 1];
2477  }
2478  }
2479 
2480  kernel.packet[0] = pload<Packet16h>(out[0]);
2481  kernel.packet[1] = pload<Packet16h>(out[1]);
2482  kernel.packet[2] = pload<Packet16h>(out[2]);
2483  kernel.packet[3] = pload<Packet16h>(out[3]);
2484  kernel.packet[4] = pload<Packet16h>(out[4]);
2485  kernel.packet[5] = pload<Packet16h>(out[5]);
2486  kernel.packet[6] = pload<Packet16h>(out[6]);
2487  kernel.packet[7] = pload<Packet16h>(out[7]);
2488 }
2489 
2490 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 4>& kernel) {
2491  EIGEN_ALIGN64 half in[4][16];
2492  pstore<half>(in[0], kernel.packet[0]);
2493  pstore<half>(in[1], kernel.packet[1]);
2494  pstore<half>(in[2], kernel.packet[2]);
2495  pstore<half>(in[3], kernel.packet[3]);
2496 
2497  EIGEN_ALIGN64 half out[4][16];
2498 
2499  for (int i = 0; i < 4; ++i) {
2500  for (int j = 0; j < 4; ++j) {
2501  out[i][j] = in[j][4 * i];
2502  }
2503  for (int j = 0; j < 4; ++j) {
2504  out[i][j + 4] = in[j][4 * i + 1];
2505  }
2506  for (int j = 0; j < 4; ++j) {
2507  out[i][j + 8] = in[j][4 * i + 2];
2508  }
2509  for (int j = 0; j < 4; ++j) {
2510  out[i][j + 12] = in[j][4 * i + 3];
2511  }
2512  }
2513 
2514  kernel.packet[0] = pload<Packet16h>(out[0]);
2515  kernel.packet[1] = pload<Packet16h>(out[1]);
2516  kernel.packet[2] = pload<Packet16h>(out[2]);
2517  kernel.packet[3] = pload<Packet16h>(out[3]);
2518 }
2519 
2520 #endif // EIGEN_VECTORIZE_AVX512FP16
2521 
2522 template <>
2523 struct is_arithmetic<Packet16bf> {
2524  enum { value = true };
2525 };
2526 
2527 template <>
2528 struct packet_traits<bfloat16> : default_packet_traits {
2529  typedef Packet16bf type;
2530  typedef Packet8bf half;
2531  enum {
2532  Vectorizable = 1,
2533  AlignedOnScalar = 1,
2534  size = 16,
2535  HasBlend = 0,
2536  HasInsert = 1,
2537  HasSin = EIGEN_FAST_MATH,
2538  HasCos = EIGEN_FAST_MATH,
2539  HasSqrt = 1,
2540  HasRsqrt = 1,
2541 #ifdef EIGEN_VECTORIZE_AVX512DQ
2542  HasLog = 1, // Currently fails test with bad accuracy.
2543  HasLog1p = 1,
2544  HasExpm1 = 1,
2545  HasNdtri = 1,
2546  HasBessel = 1,
2547 #endif
2548  HasExp = 1,
2549  HasTanh = EIGEN_FAST_MATH,
2550  HasErf = EIGEN_FAST_MATH,
2551  HasCmp = 1,
2552  HasDiv = 1
2553  };
2554 };
2555 
2556 template <>
2557 struct unpacket_traits<Packet16bf> {
2558  typedef bfloat16 type;
2559  enum {
2560  size = 16,
2561  alignment = Aligned32,
2562  vectorizable = true,
2563  masked_load_available = false,
2564  masked_store_available = false
2565  };
2566  typedef Packet8bf half;
2567 };
2568 
2569 template <>
2570 EIGEN_STRONG_INLINE Packet16bf pset1<Packet16bf>(const bfloat16& from) {
2571  return _mm256_set1_epi16(from.value);
2572 }
2573 
2574 template <>
2575 EIGEN_STRONG_INLINE bfloat16 pfirst<Packet16bf>(const Packet16bf& from) {
2576  bfloat16 t;
2577  t.value = static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
2578  return t;
2579 }
2580 
2581 template <>
2582 EIGEN_STRONG_INLINE Packet16bf pload<Packet16bf>(const bfloat16* from) {
2583  return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2584 }
2585 
2586 template <>
2587 EIGEN_STRONG_INLINE Packet16bf ploadu<Packet16bf>(const bfloat16* from) {
2588  return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2589 }
2590 
2591 template <>
2592 EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet16bf& from) {
2593  EIGEN_DEBUG_ALIGNED_STORE
2594  _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
2595 }
2596 
2597 template <>
2598 EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet16bf& from) {
2599  EIGEN_DEBUG_UNALIGNED_STORE
2600  _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
2601 }
2602 
2603 template <>
2604 EIGEN_STRONG_INLINE Packet16bf ploaddup<Packet16bf>(const bfloat16* from) {
2605  unsigned short a = from[0].value;
2606  unsigned short b = from[1].value;
2607  unsigned short c = from[2].value;
2608  unsigned short d = from[3].value;
2609  unsigned short e = from[4].value;
2610  unsigned short f = from[5].value;
2611  unsigned short g = from[6].value;
2612  unsigned short h = from[7].value;
2613  return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2614 }
2615 
2616 template <>
2617 EIGEN_STRONG_INLINE Packet16bf ploadquad(const bfloat16* from) {
2618  unsigned short a = from[0].value;
2619  unsigned short b = from[1].value;
2620  unsigned short c = from[2].value;
2621  unsigned short d = from[3].value;
2622  return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2623 }
2624 
2625 EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) {
2626  return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
2627 }
2628 
2629 // Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm.
2630 EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) {
2631  Packet16bf r;
2632 
2633 #if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_STRICT_AT_LEAST(10, 1, 0)
2634  // Since GCC 10.1 supports avx512bf16 and C style explicit cast
2635  // (C++ static_cast is not supported yet), do conversion via intrinsic
2636  // and register path for performance.
2637  r = (__m256i)(_mm512_cvtneps_pbh(a));
2638 
2639 #else
2640  __m512i t;
2641  __m512i input = _mm512_castps_si512(a);
2642  __m512i nan = _mm512_set1_epi32(0x7fc0);
2643 
2644  // uint32_t lsb = (input >> 16) & 1;
2645  t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
2646  // uint32_t rounding_bias = 0x7fff + lsb;
2647  t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
2648  // input += rounding_bias;
2649  t = _mm512_add_epi32(t, input);
2650  // input = input >> 16;
2651  t = _mm512_srli_epi32(t, 16);
2652 
2653  // Check NaN before converting back to bf16
2654  __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
2655 
2656  t = _mm512_mask_blend_epi32(mask, nan, t);
2657  // output.value = static_cast<uint16_t>(input);
2658  r = _mm512_cvtepi32_epi16(t);
2659 #endif // EIGEN_VECTORIZE_AVX512BF16
2660 
2661  return r;
2662 }
2663 
2664 template <>
2665 EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) {
2666  return Packet16bf(ptrue<Packet8i>(Packet8i(a)));
2667 }
2668 
2669 template <>
2670 EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) {
2671  return Packet16bf(por<Packet8i>(Packet8i(a), Packet8i(b)));
2672 }
2673 
2674 template <>
2675 EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) {
2676  return Packet16bf(pxor<Packet8i>(Packet8i(a), Packet8i(b)));
2677 }
2678 
2679 template <>
2680 EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) {
2681  return Packet16bf(pand<Packet8i>(Packet8i(a), Packet8i(b)));
2682 }
2683 
2684 template <>
2685 EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a, const Packet16bf& b) {
2686  return Packet16bf(pandnot<Packet8i>(Packet8i(a), Packet8i(b)));
2687 }
2688 
2689 template <>
2690 EIGEN_STRONG_INLINE Packet16bf pselect(const Packet16bf& mask, const Packet16bf& a, const Packet16bf& b) {
2691  // Input mask is expected to be all 0/1, handle it with 8-bit
2692  // intrinsic for performance.
2693  return _mm256_blendv_epi8(b, a, mask);
2694 }
2695 
2696 template <>
2697 EIGEN_STRONG_INLINE Packet16bf pround<Packet16bf>(const Packet16bf& a) {
2698  return F32ToBf16(pround<Packet16f>(Bf16ToF32(a)));
2699 }
2700 
2701 template <>
2702 EIGEN_STRONG_INLINE Packet16bf print<Packet16bf>(const Packet16bf& a) {
2703  return F32ToBf16(print<Packet16f>(Bf16ToF32(a)));
2704 }
2705 
2706 template <>
2707 EIGEN_STRONG_INLINE Packet16bf pceil<Packet16bf>(const Packet16bf& a) {
2708  return F32ToBf16(pceil<Packet16f>(Bf16ToF32(a)));
2709 }
2710 
2711 template <>
2712 EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(const Packet16bf& a) {
2713  return F32ToBf16(pfloor<Packet16f>(Bf16ToF32(a)));
2714 }
2715 
2716 template <>
2717 EIGEN_STRONG_INLINE Packet16bf ptrunc<Packet16bf>(const Packet16bf& a) {
2718  return F32ToBf16(ptrunc<Packet16f>(Bf16ToF32(a)));
2719 }
2720 
2721 template <>
2722 EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a, const Packet16bf& b) {
2723  return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2724 }
2725 
2726 template <>
2727 EIGEN_STRONG_INLINE Packet16bf pcmp_le(const Packet16bf& a, const Packet16bf& b) {
2728  return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2729 }
2730 
2731 template <>
2732 EIGEN_STRONG_INLINE Packet16bf pcmp_lt(const Packet16bf& a, const Packet16bf& b) {
2733  return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2734 }
2735 
2736 template <>
2737 EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(const Packet16bf& a, const Packet16bf& b) {
2738  return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2739 }
2740 
2741 template <>
2742 EIGEN_STRONG_INLINE Packet16bf pnegate(const Packet16bf& a) {
2743  Packet16bf sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2744  return _mm256_xor_si256(a, sign_mask);
2745 }
2746 
2747 template <>
2748 EIGEN_STRONG_INLINE Packet16bf pconj(const Packet16bf& a) {
2749  return a;
2750 }
2751 
2752 template <>
2753 EIGEN_STRONG_INLINE Packet16bf pabs(const Packet16bf& a) {
2754  const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
2755  return _mm256_andnot_si256(sign_mask, a);
2756 }
2757 
2758 template <>
2759 EIGEN_STRONG_INLINE Packet16bf padd<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2760  return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2761 }
2762 
2763 template <>
2764 EIGEN_STRONG_INLINE Packet16bf psub<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2765  return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2766 }
2767 
2768 template <>
2769 EIGEN_STRONG_INLINE Packet16bf pmul<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2770  return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b)));
2771 }
2772 
2773 template <>
2774 EIGEN_STRONG_INLINE Packet16bf pmadd<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2775  return F32ToBf16(pmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2776 }
2777 
2778 template <>
2779 EIGEN_STRONG_INLINE Packet16bf pmsub<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2780  return F32ToBf16(pmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2781 }
2782 
2783 template <>
2784 EIGEN_STRONG_INLINE Packet16bf pnmadd<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2785  return F32ToBf16(pnmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2786 }
2787 
2788 template <>
2789 EIGEN_STRONG_INLINE Packet16bf pnmsub<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2790  return F32ToBf16(pnmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2791 }
2792 
2793 template <>
2794 EIGEN_STRONG_INLINE Packet16bf pdiv<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2795  return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2796 }
2797 
2798 template <>
2799 EIGEN_STRONG_INLINE Packet16bf pmin<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2800  return F32ToBf16(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2801 }
2802 
2803 template <>
2804 EIGEN_STRONG_INLINE Packet16bf pmax<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2805  return F32ToBf16(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2806 }
2807 
2808 template <>
2809 EIGEN_STRONG_INLINE Packet16bf plset<Packet16bf>(const bfloat16& a) {
2810  return F32ToBf16(plset<Packet16f>(static_cast<float>(a)));
2811 }
2812 
2813 template <>
2814 EIGEN_STRONG_INLINE Packet8bf predux_half_dowto4<Packet16bf>(const Packet16bf& a) {
2815  Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
2816  Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
2817  return padd<Packet8bf>(lane0, lane1);
2818 }
2819 
2820 template <>
2821 EIGEN_STRONG_INLINE Packet16bf preverse(const Packet16bf& a) {
2822  __m256i m = _mm256_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7,
2823  4, 5, 2, 3, 0, 1);
2824 
2825  Packet16bf res;
2826  // Swap hi and lo first because shuffle is in 128-bit lanes.
2827  res = _mm256_permute2x128_si256(a, a, 1);
2828  // Shuffle 8-bit values in src within 2*128-bit lanes.
2829  return _mm256_shuffle_epi8(res, m);
2830 }
2831 
2832 template <>
2833 EIGEN_STRONG_INLINE Packet16bf pgather<bfloat16, Packet16bf>(const bfloat16* from, Index stride) {
2834  return _mm256_set_epi16(
2835  from[15 * stride].value, from[14 * stride].value, from[13 * stride].value, from[12 * stride].value,
2836  from[11 * stride].value, from[10 * stride].value, from[9 * stride].value, from[8 * stride].value,
2837  from[7 * stride].value, from[6 * stride].value, from[5 * stride].value, from[4 * stride].value,
2838  from[3 * stride].value, from[2 * stride].value, from[1 * stride].value, from[0 * stride].value);
2839 }
2840 
2841 template <>
2842 EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet16bf>(bfloat16* to, const Packet16bf& from, Index stride) {
2843  EIGEN_ALIGN64 bfloat16 aux[16];
2844  pstore(aux, from);
2845  to[stride * 0] = aux[0];
2846  to[stride * 1] = aux[1];
2847  to[stride * 2] = aux[2];
2848  to[stride * 3] = aux[3];
2849  to[stride * 4] = aux[4];
2850  to[stride * 5] = aux[5];
2851  to[stride * 6] = aux[6];
2852  to[stride * 7] = aux[7];
2853  to[stride * 8] = aux[8];
2854  to[stride * 9] = aux[9];
2855  to[stride * 10] = aux[10];
2856  to[stride * 11] = aux[11];
2857  to[stride * 12] = aux[12];
2858  to[stride * 13] = aux[13];
2859  to[stride * 14] = aux[14];
2860  to[stride * 15] = aux[15];
2861 }
2862 
2863 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf, 16>& kernel) {
2864  __m256i a = kernel.packet[0];
2865  __m256i b = kernel.packet[1];
2866  __m256i c = kernel.packet[2];
2867  __m256i d = kernel.packet[3];
2868  __m256i e = kernel.packet[4];
2869  __m256i f = kernel.packet[5];
2870  __m256i g = kernel.packet[6];
2871  __m256i h = kernel.packet[7];
2872  __m256i i = kernel.packet[8];
2873  __m256i j = kernel.packet[9];
2874  __m256i k = kernel.packet[10];
2875  __m256i l = kernel.packet[11];
2876  __m256i m = kernel.packet[12];
2877  __m256i n = kernel.packet[13];
2878  __m256i o = kernel.packet[14];
2879  __m256i p = kernel.packet[15];
2880 
2881  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2882  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2883  __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2884  __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2885  __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2886  __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2887  __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2888  __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2889 
2890  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2891  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2892  __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2893  __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2894  __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2895  __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2896  __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2897  __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2898 
2899  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2900  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2901  __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2902  __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2903  __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2904  __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2905  __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2906  __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2907 
2908  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2909  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2910  __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2911  __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2912  __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2913  __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2914  __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2915  __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2916 
2917  __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2918  __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2919  __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2920  __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2921  __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2922  __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2923  __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2924  __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2925  __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2926  __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2927  __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2928  __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2929  __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2930  __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2931  __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2932  __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2933 
2934  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2935  kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2936  kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2937  kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2938  kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2939  kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2940  kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2941  kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2942  kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2943  kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2944  kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2945  kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2946  kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2947  kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2948  kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2949  kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2950  kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2951 }
2952 
2953 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf, 4>& kernel) {
2954  __m256i a = kernel.packet[0];
2955  __m256i b = kernel.packet[1];
2956  __m256i c = kernel.packet[2];
2957  __m256i d = kernel.packet[3];
2958 
2959  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2960  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2961  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2962  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2963 
2964  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2965  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2966  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2967  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2968 
2969  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2970  kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
2971  kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
2972  kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
2973  kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
2974 }
2975 
2976 // Minimal implementation of 16-bit int packets for use in pfrexp, pldexp.
2977 
2978 template <>
2979 EIGEN_STRONG_INLINE Packet32s pset1<Packet32s>(const numext::int16_t& x) {
2980  return _mm512_set1_epi16(x);
2981 }
2982 
2983 template <>
2984 EIGEN_STRONG_INLINE Packet16s pset1<Packet16s>(const numext::int16_t& x) {
2985  return _mm256_set1_epi16(x);
2986 }
2987 
2988 template <>
2989 EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const numext::int16_t& x) {
2990  return _mm_set1_epi16(x);
2991 }
2992 
2993 template <>
2994 EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
2995  EIGEN_DEBUG_ALIGNED_STORE
2996  _mm512_store_epi32(out, x);
2997 }
2998 
2999 template <>
3000 EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
3001  EIGEN_DEBUG_ALIGNED_STORE
3002 #if defined(EIGEN_VECTORIZE_AVX512F) && defined(EIGEN_VECTORIZE_AVX512VL)
3003  _mm256_store_epi32(out, x);
3004 #else
3005  _mm256_store_si256(reinterpret_cast<__m256i*>(out), x);
3006 #endif
3007 }
3008 
3009 template <>
3010 EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
3011  EIGEN_DEBUG_ALIGNED_STORE
3012 #if defined(EIGEN_VECTORIZE_AVX512F) && defined(EIGEN_VECTORIZE_AVX512VL)
3013  _mm256_store_epi32(out, x);
3014 #else
3015  _mm_store_si128(reinterpret_cast<__m128i*>(out), x);
3016 #endif
3017 }
3018 
3019 template <>
3020 EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
3021  EIGEN_DEBUG_UNALIGNED_STORE
3022  _mm512_storeu_epi32(out, x);
3023 }
3024 
3025 template <>
3026 EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
3027  EIGEN_DEBUG_UNALIGNED_STORE
3028  _mm256_storeu_epi32(out, x);
3029 }
3030 
3031 template <>
3032 EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
3033  EIGEN_DEBUG_UNALIGNED_STORE
3034  _mm_storeu_epi32(out, x);
3035 }
3036 
3037 template <>
3038 EIGEN_STRONG_INLINE Packet32s padd(const Packet32s& a, const Packet32s& b) {
3039  return _mm512_add_epi16(a, b);
3040 }
3041 
3042 template <>
3043 EIGEN_STRONG_INLINE Packet16s padd(const Packet16s& a, const Packet16s& b) {
3044  return _mm256_add_epi16(a, b);
3045 }
3046 
3047 template <>
3048 EIGEN_STRONG_INLINE Packet8s padd(const Packet8s& a, const Packet8s& b) {
3049  return _mm_add_epi16(a, b);
3050 }
3051 
3052 template <>
3053 EIGEN_STRONG_INLINE Packet32s psub(const Packet32s& a, const Packet32s& b) {
3054  return _mm512_sub_epi16(a, b);
3055 }
3056 
3057 template <>
3058 EIGEN_STRONG_INLINE Packet16s psub(const Packet16s& a, const Packet16s& b) {
3059  return _mm256_sub_epi16(a, b);
3060 }
3061 
3062 template <>
3063 EIGEN_STRONG_INLINE Packet8s psub(const Packet8s& a, const Packet8s& b) {
3064  return _mm_sub_epi16(a, b);
3065 }
3066 
3067 template <>
3068 EIGEN_STRONG_INLINE Packet32s pmul(const Packet32s& a, const Packet32s& b) {
3069  return _mm512_mullo_epi16(a, b);
3070 }
3071 
3072 template <>
3073 EIGEN_STRONG_INLINE Packet16s pmul(const Packet16s& a, const Packet16s& b) {
3074  return _mm256_mullo_epi16(a, b);
3075 }
3076 
3077 template <>
3078 EIGEN_STRONG_INLINE Packet8s pmul(const Packet8s& a, const Packet8s& b) {
3079  return _mm_mullo_epi16(a, b);
3080 }
3081 
3082 template <>
3083 EIGEN_STRONG_INLINE Packet32s pnegate(const Packet32s& a) {
3084  return _mm512_sub_epi16(_mm512_setzero_si512(), a);
3085 }
3086 
3087 template <>
3088 EIGEN_STRONG_INLINE Packet16s pnegate(const Packet16s& a) {
3089  return _mm256_sub_epi16(_mm256_setzero_si256(), a);
3090 }
3091 
3092 template <>
3093 EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
3094  return _mm_sub_epi16(_mm_setzero_si128(), a);
3095 }
3096 
3097 template <int N>
3098 EIGEN_STRONG_INLINE Packet32s parithmetic_shift_right(Packet32s a) {
3099  return _mm512_srai_epi16(a, N);
3100 }
3101 
3102 template <int N>
3103 EIGEN_STRONG_INLINE Packet16s parithmetic_shift_right(Packet16s a) {
3104  return _mm256_srai_epi16(a, N);
3105 }
3106 
3107 template <int N>
3108 EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) {
3109  return _mm_srai_epi16(a, N);
3110 }
3111 
3112 template <int N>
3113 EIGEN_STRONG_INLINE Packet32s plogical_shift_left(Packet32s a) {
3114  return _mm512_slli_epi16(a, N);
3115 }
3116 
3117 template <int N>
3118 EIGEN_STRONG_INLINE Packet16s plogical_shift_left(Packet16s a) {
3119  return _mm256_slli_epi16(a, N);
3120 }
3121 
3122 template <int N>
3123 EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) {
3124  return _mm_slli_epi16(a, N);
3125 }
3126 
3127 template <int N>
3128 EIGEN_STRONG_INLINE Packet32s plogical_shift_right(Packet32s a) {
3129  return _mm512_srli_epi16(a, N);
3130 }
3131 
3132 template <int N>
3133 EIGEN_STRONG_INLINE Packet16s plogical_shift_right(Packet16s a) {
3134  return _mm256_srli_epi16(a, N);
3135 }
3136 
3137 template <int N>
3138 EIGEN_STRONG_INLINE Packet8s plogical_shift_right(Packet8s a) {
3139  return _mm_srli_epi16(a, N);
3140 }
3141 
3142 } // end namespace internal
3143 
3144 } // end namespace Eigen
3145 
3146 #endif // EIGEN_PACKET_MATH_AVX512_H
Eigen::Aligned16
Definition: Constants.h:237
Eigen
Namespace containing all symbols from the Eigen library.
Definition: B01_Experimental.dox:1
Eigen::Aligned32
Definition: Constants.h:238
Eigen::Aligned64
Definition: Constants.h:239
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:82
Generated by doxygen 1.8.14