Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
elpa
elpa
Commits
863d1b97
Commit
863d1b97
authored
Jun 12, 2017
by
Andreas Marek
Browse files
Start to unfiy complex single/double SSE block 1 kernel
parent
dc90a3b8
Changes
5
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
Makefile.am
View file @
863d1b97
...
...
@@ -595,6 +595,7 @@ EXTRA_DIST = \
src/elpa2/kernels/elpa2_kernels_real_avx-avx2_2hv_template.Xc
\
src/elpa2/kernels/elpa2_kernels_real_avx-avx2_4hv_template.Xc
\
src/elpa2/kernels/elpa2_kernels_real_avx-avx2_6hv_template.Xc
\
src/elpa2/kernels/elpa2_kernels_complex_sse_1hv_template.Xc
\
src/elpa2/redist_band.X90
\
src/elpa2/pack_unpack_cpu.X90
\
src/elpa2/pack_unpack_gpu.X90
\
...
...
src/elpa2/kernels/elpa2_kernels_complex_sse_1hv_double_precision.c
View file @
863d1b97
...
...
@@ -42,514 +42,14 @@
// any derivatives of ELPA under the same license that we chose for
// the original distribution, the GNU Lesser General Public License.
//
//
// --------------------------------------------------------------------------------------------------
//
// This file contains the compute intensive kernels for the Householder transformations.
// It should be compiled with the highest possible optimization level.
//
// On Intel Nehalem or Intel Westmere or AMD Magny Cours use -O3 -msse3
// On Intel Sandy Bridge use -O3 -mavx
//
// Copyright of the original code rests with the authors inside the ELPA
// consortium. The copyright of any additional modifications shall rest
// with their original authors, but shall adhere to the licensing terms
// distributed along with the original code in the file "COPYING".
//
// Author: Alexander Heinecke (alexander.heinecke@mytum.de)
// Adapted for building a shared-library by Andreas Marek, MPCDF (andreas.marek@mpcdf.mpg.de)
// --------------------------------------------------------------------------------------------------
// Author: Andreas Marek, MPCDF
#include "config-f90.h"
#include <complex.h>
#include <x86intrin.h>
#define __forceinline __attribute__((always_inline))
#ifdef HAVE_SSE_INTRINSICS
#undef __AVX__
#endif
//Forward declaration
static
__forceinline
void
hh_trafo_complex_kernel_6_SSE_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
nb
,
int
ldq
);
static
__forceinline
void
hh_trafo_complex_kernel_4_SSE_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
nb
,
int
ldq
);
static
__forceinline
void
hh_trafo_complex_kernel_2_SSE_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
nb
,
int
ldq
);
/*
!f>#ifdef WITH_COMPLEX_SSE_BLOCK1_KERNEL
!f> interface
!f> subroutine single_hh_trafo_complex_sse_1hv_double(q, hh, pnb, pnq, pldq) &
!f> bind(C, name="single_hh_trafo_complex_sse_1hv_double")
!f> use, intrinsic :: iso_c_binding
!f> integer(kind=c_int) :: pnb, pnq, pldq
!f> ! complex(kind=c_double_complex) :: q(*)
!f> type(c_ptr), value :: q
!f> complex(kind=c_double_complex) :: hh(pnb,2)
!f> end subroutine
!f> end interface
!f>#endif
*/
void
single_hh_trafo_complex_sse_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
*
pnb
,
int
*
pnq
,
int
*
pldq
)
{
int
i
;
int
nb
=
*
pnb
;
int
nq
=
*
pldq
;
int
ldq
=
*
pldq
;
//int ldh = *pldh;
for
(
i
=
0
;
i
<
nq
-
4
;
i
+=
6
)
{
hh_trafo_complex_kernel_6_SSE_1hv_double
(
&
q
[
i
],
hh
,
nb
,
ldq
);
}
if
(
nq
-
i
==
0
)
{
return
;
}
else
{
if
(
nq
-
i
>
2
)
{
hh_trafo_complex_kernel_4_SSE_1hv_double
(
&
q
[
i
],
hh
,
nb
,
ldq
);
}
else
{
hh_trafo_complex_kernel_2_SSE_1hv_double
(
&
q
[
i
],
hh
,
nb
,
ldq
);
}
}
}
static
__forceinline
void
hh_trafo_complex_kernel_6_SSE_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
nb
,
int
ldq
)
{
double
*
q_dbl
=
(
double
*
)
q
;
double
*
hh_dbl
=
(
double
*
)
hh
;
__m128d
x1
,
x2
,
x3
,
x4
,
x5
,
x6
;
__m128d
q1
,
q2
,
q3
,
q4
,
q5
,
q6
;
__m128d
h1_real
,
h1_imag
;
__m128d
tmp1
,
tmp2
,
tmp3
,
tmp4
,
tmp5
,
tmp6
;
int
i
=
0
;
__m128d
sign
=
(
__m128d
)
_mm_set_epi64x
(
0x8000000000000000
,
0x8000000000000000
);
x1
=
_mm_load_pd
(
&
q_dbl
[
0
]);
x2
=
_mm_load_pd
(
&
q_dbl
[
2
]);
x3
=
_mm_load_pd
(
&
q_dbl
[
4
]);
x4
=
_mm_load_pd
(
&
q_dbl
[
6
]);
x5
=
_mm_load_pd
(
&
q_dbl
[
8
]);
x6
=
_mm_load_pd
(
&
q_dbl
[
10
]);
for
(
i
=
1
;
i
<
nb
;
i
++
)
{
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
i
*
2
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[(
i
*
2
)
+
1
]);
#ifndef __ELPA_USE_FMA__
// conjugate
h1_imag
=
_mm_xor_pd
(
h1_imag
,
sign
);
#endif
q1
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
]);
q3
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
4
]);
q4
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
6
]);
q5
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
8
]);
q6
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
10
]);
tmp1
=
_mm_mul_pd
(
h1_imag
,
q1
);
#ifdef __ELPA_USE_FMA__
x1
=
_mm_add_pd
(
x1
,
_mm_msubadd_pd
(
h1_real
,
q1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x1
=
_mm_add_pd
(
x1
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
q2
);
#ifdef __ELPA_USE_FMA__
x2
=
_mm_add_pd
(
x2
,
_mm_msubadd_pd
(
h1_real
,
q2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x2
=
_mm_add_pd
(
x2
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp3
=
_mm_mul_pd
(
h1_imag
,
q3
);
#ifdef __ELPA_USE_FMA__
x3
=
_mm_add_pd
(
x3
,
_mm_msubadd_pd
(
h1_real
,
q3
,
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x3
=
_mm_add_pd
(
x3
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q3
),
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp4
=
_mm_mul_pd
(
h1_imag
,
q4
);
#ifdef __ELPA_USE_FMA__
x4
=
_mm_add_pd
(
x4
,
_mm_msubadd_pd
(
h1_real
,
q4
,
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x4
=
_mm_add_pd
(
x4
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q4
),
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp5
=
_mm_mul_pd
(
h1_imag
,
q5
);
#ifdef __ELPA_USE_FMA__
x5
=
_mm_add_pd
(
x5
,
_mm_msubadd_pd
(
h1_real
,
q5
,
_mm_shuffle_pd
(
tmp5
,
tmp5
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x5
=
_mm_add_pd
(
x5
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q5
),
_mm_shuffle_pd
(
tmp5
,
tmp5
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp6
=
_mm_mul_pd
(
h1_imag
,
q6
);
#ifdef __ELPA_USE_FMA__
x6
=
_mm_add_pd
(
x6
,
_mm_msubadd_pd
(
h1_real
,
q6
,
_mm_shuffle_pd
(
tmp6
,
tmp6
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x6
=
_mm_add_pd
(
x6
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q6
),
_mm_shuffle_pd
(
tmp6
,
tmp6
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
}
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
0
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[
1
]);
h1_real
=
_mm_xor_pd
(
h1_real
,
sign
);
h1_imag
=
_mm_xor_pd
(
h1_imag
,
sign
);
tmp1
=
_mm_mul_pd
(
h1_imag
,
x1
);
#ifdef __ELPA_USE_FMA__
x1
=
_mm_maddsub_pd
(
h1_real
,
x1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x1
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
x2
);
#ifdef __ELPA_USE_FMA__
x2
=
_mm_maddsub_pd
(
h1_real
,
x2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x2
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp3
=
_mm_mul_pd
(
h1_imag
,
x3
);
#ifdef __ELPA_USE_FMA__
x3
=
_mm_maddsub_pd
(
h1_real
,
x3
,
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x3
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x3
),
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp4
=
_mm_mul_pd
(
h1_imag
,
x4
);
#ifdef __ELPA_USE_FMA__
x4
=
_mm_maddsub_pd
(
h1_real
,
x4
,
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x4
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x4
),
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp5
=
_mm_mul_pd
(
h1_imag
,
x5
);
#ifdef __ELPA_USE_FMA__
x5
=
_mm_maddsub_pd
(
h1_real
,
x5
,
_mm_shuffle_pd
(
tmp5
,
tmp5
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x5
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x5
),
_mm_shuffle_pd
(
tmp5
,
tmp5
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp6
=
_mm_mul_pd
(
h1_imag
,
x6
);
#ifdef __ELPA_USE_FMA__
x6
=
_mm_maddsub_pd
(
h1_real
,
x6
,
_mm_shuffle_pd
(
tmp6
,
tmp6
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x6
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x6
),
_mm_shuffle_pd
(
tmp6
,
tmp6
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
q1
=
_mm_load_pd
(
&
q_dbl
[
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[
2
]);
q3
=
_mm_load_pd
(
&
q_dbl
[
4
]);
q4
=
_mm_load_pd
(
&
q_dbl
[
6
]);
q5
=
_mm_load_pd
(
&
q_dbl
[
8
]);
q6
=
_mm_load_pd
(
&
q_dbl
[
10
]);
q1
=
_mm_add_pd
(
q1
,
x1
);
q2
=
_mm_add_pd
(
q2
,
x2
);
q3
=
_mm_add_pd
(
q3
,
x3
);
q4
=
_mm_add_pd
(
q4
,
x4
);
q5
=
_mm_add_pd
(
q5
,
x5
);
q6
=
_mm_add_pd
(
q6
,
x6
);
_mm_store_pd
(
&
q_dbl
[
0
],
q1
);
_mm_store_pd
(
&
q_dbl
[
2
],
q2
);
_mm_store_pd
(
&
q_dbl
[
4
],
q3
);
_mm_store_pd
(
&
q_dbl
[
6
],
q4
);
_mm_store_pd
(
&
q_dbl
[
8
],
q5
);
_mm_store_pd
(
&
q_dbl
[
10
],
q6
);
for
(
i
=
1
;
i
<
nb
;
i
++
)
{
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
i
*
2
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[(
i
*
2
)
+
1
]);
q1
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
]);
q3
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
4
]);
q4
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
6
]);
q5
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
8
]);
q6
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
10
]);
tmp1
=
_mm_mul_pd
(
h1_imag
,
x1
);
#ifdef __ELPA_USE_FMA__
q1
=
_mm_add_pd
(
q1
,
_mm_maddsub_pd
(
h1_real
,
x1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q1
=
_mm_add_pd
(
q1
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
x2
);
#ifdef __ELPA_USE_FMA__
q2
=
_mm_add_pd
(
q2
,
_mm_maddsub_pd
(
h1_real
,
x2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q2
=
_mm_add_pd
(
q2
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp3
=
_mm_mul_pd
(
h1_imag
,
x3
);
#ifdef __ELPA_USE_FMA__
q3
=
_mm_add_pd
(
q3
,
_mm_maddsub_pd
(
h1_real
,
x3
,
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q3
=
_mm_add_pd
(
q3
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x3
),
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp4
=
_mm_mul_pd
(
h1_imag
,
x4
);
#ifdef __ELPA_USE_FMA__
q4
=
_mm_add_pd
(
q4
,
_mm_maddsub_pd
(
h1_real
,
x4
,
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q4
=
_mm_add_pd
(
q4
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x4
),
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp5
=
_mm_mul_pd
(
h1_imag
,
x5
);
#ifdef __ELPA_USE_FMA__
q5
=
_mm_add_pd
(
q5
,
_mm_maddsub_pd
(
h1_real
,
x5
,
_mm_shuffle_pd
(
tmp5
,
tmp5
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q5
=
_mm_add_pd
(
q5
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x5
),
_mm_shuffle_pd
(
tmp5
,
tmp5
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp6
=
_mm_mul_pd
(
h1_imag
,
x6
);
#ifdef __ELPA_USE_FMA__
q6
=
_mm_add_pd
(
q6
,
_mm_maddsub_pd
(
h1_real
,
x6
,
_mm_shuffle_pd
(
tmp6
,
tmp6
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q6
=
_mm_add_pd
(
q6
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x6
),
_mm_shuffle_pd
(
tmp6
,
tmp6
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
],
q1
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
],
q2
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
4
],
q3
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
6
],
q4
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
8
],
q5
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
10
],
q6
);
}
}
static
__forceinline
void
hh_trafo_complex_kernel_4_SSE_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
nb
,
int
ldq
)
{
double
*
q_dbl
=
(
double
*
)
q
;
double
*
hh_dbl
=
(
double
*
)
hh
;
__m128d
x1
,
x2
,
x3
,
x4
;
__m128d
q1
,
q2
,
q3
,
q4
;
__m128d
h1_real
,
h1_imag
;
__m128d
tmp1
,
tmp2
,
tmp3
,
tmp4
;
int
i
=
0
;
__m128d
sign
=
(
__m128d
)
_mm_set_epi64x
(
0x8000000000000000
,
0x8000000000000000
);
x1
=
_mm_load_pd
(
&
q_dbl
[
0
]);
x2
=
_mm_load_pd
(
&
q_dbl
[
2
]);
x3
=
_mm_load_pd
(
&
q_dbl
[
4
]);
x4
=
_mm_load_pd
(
&
q_dbl
[
6
]);
for
(
i
=
1
;
i
<
nb
;
i
++
)
{
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
i
*
2
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[(
i
*
2
)
+
1
]);
#ifndef __ELPA_USE_FMA__
// conjugate
h1_imag
=
_mm_xor_pd
(
h1_imag
,
sign
);
#endif
q1
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
]);
q3
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
4
]);
q4
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
6
]);
tmp1
=
_mm_mul_pd
(
h1_imag
,
q1
);
#ifdef __ELPA_USE_FMA__
x1
=
_mm_add_pd
(
x1
,
_mm_msubadd_pd
(
h1_real
,
q1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x1
=
_mm_add_pd
(
x1
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
q2
);
#ifdef __ELPA_USE_FMA__
x2
=
_mm_add_pd
(
x2
,
_mm_msubadd_pd
(
h1_real
,
q2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x2
=
_mm_add_pd
(
x2
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp3
=
_mm_mul_pd
(
h1_imag
,
q3
);
#ifdef __ELPA_USE_FMA__
x3
=
_mm_add_pd
(
x3
,
_mm_msubadd_pd
(
h1_real
,
q3
,
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x3
=
_mm_add_pd
(
x3
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q3
),
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp4
=
_mm_mul_pd
(
h1_imag
,
q4
);
#ifdef __ELPA_USE_FMA__
x4
=
_mm_add_pd
(
x4
,
_mm_msubadd_pd
(
h1_real
,
q4
,
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x4
=
_mm_add_pd
(
x4
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q4
),
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
}
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
0
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[
1
]);
h1_real
=
_mm_xor_pd
(
h1_real
,
sign
);
h1_imag
=
_mm_xor_pd
(
h1_imag
,
sign
);
tmp1
=
_mm_mul_pd
(
h1_imag
,
x1
);
#ifdef __ELPA_USE_FMA__
x1
=
_mm_maddsub_pd
(
h1_real
,
x1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x1
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
x2
);
#ifdef __ELPA_USE_FMA__
x2
=
_mm_maddsub_pd
(
h1_real
,
x2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x2
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp3
=
_mm_mul_pd
(
h1_imag
,
x3
);
#ifdef __ELPA_USE_FMA__
x3
=
_mm_maddsub_pd
(
h1_real
,
x3
,
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x3
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x3
),
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp4
=
_mm_mul_pd
(
h1_imag
,
x4
);
#ifdef __ELPA_USE_FMA__
x4
=
_mm_maddsub_pd
(
h1_real
,
x4
,
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x4
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x4
),
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
q1
=
_mm_load_pd
(
&
q_dbl
[
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[
2
]);
q3
=
_mm_load_pd
(
&
q_dbl
[
4
]);
q4
=
_mm_load_pd
(
&
q_dbl
[
6
]);
q1
=
_mm_add_pd
(
q1
,
x1
);
q2
=
_mm_add_pd
(
q2
,
x2
);
q3
=
_mm_add_pd
(
q3
,
x3
);
q4
=
_mm_add_pd
(
q4
,
x4
);
_mm_store_pd
(
&
q_dbl
[
0
],
q1
);
_mm_store_pd
(
&
q_dbl
[
2
],
q2
);
_mm_store_pd
(
&
q_dbl
[
4
],
q3
);
_mm_store_pd
(
&
q_dbl
[
6
],
q4
);
for
(
i
=
1
;
i
<
nb
;
i
++
)
{
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
i
*
2
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[(
i
*
2
)
+
1
]);
q1
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
]);
q3
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
4
]);
q4
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
6
]);
tmp1
=
_mm_mul_pd
(
h1_imag
,
x1
);
#ifdef __ELPA_USE_FMA__
q1
=
_mm_add_pd
(
q1
,
_mm_maddsub_pd
(
h1_real
,
x1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q1
=
_mm_add_pd
(
q1
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
x2
);
#ifdef __ELPA_USE_FMA__
q2
=
_mm_add_pd
(
q2
,
_mm_maddsub_pd
(
h1_real
,
x2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q2
=
_mm_add_pd
(
q2
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp3
=
_mm_mul_pd
(
h1_imag
,
x3
);
#ifdef __ELPA_USE_FMA__
q3
=
_mm_add_pd
(
q3
,
_mm_maddsub_pd
(
h1_real
,
x3
,
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q3
=
_mm_add_pd
(
q3
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x3
),
_mm_shuffle_pd
(
tmp3
,
tmp3
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp4
=
_mm_mul_pd
(
h1_imag
,
x4
);
#ifdef __ELPA_USE_FMA__
q4
=
_mm_add_pd
(
q4
,
_mm_maddsub_pd
(
h1_real
,
x4
,
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q4
=
_mm_add_pd
(
q4
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x4
),
_mm_shuffle_pd
(
tmp4
,
tmp4
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
],
q1
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
],
q2
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
4
],
q3
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
6
],
q4
);
}
}
static
__forceinline
void
hh_trafo_complex_kernel_2_SSE_1hv_double
(
double
complex
*
q
,
double
complex
*
hh
,
int
nb
,
int
ldq
)
{
double
*
q_dbl
=
(
double
*
)
q
;
double
*
hh_dbl
=
(
double
*
)
hh
;
__m128d
x1
,
x2
;
__m128d
q1
,
q2
;
__m128d
h1_real
,
h1_imag
;
__m128d
tmp1
,
tmp2
;
int
i
=
0
;
__m128d
sign
=
(
__m128d
)
_mm_set_epi64x
(
0x8000000000000000
,
0x8000000000000000
);
x1
=
_mm_load_pd
(
&
q_dbl
[
0
]);
x2
=
_mm_load_pd
(
&
q_dbl
[
2
]);
for
(
i
=
1
;
i
<
nb
;
i
++
)
{
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
i
*
2
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[(
i
*
2
)
+
1
]);
#ifndef __ELPA_USE_FMA__
// conjugate
h1_imag
=
_mm_xor_pd
(
h1_imag
,
sign
);
#endif
q1
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
]);
tmp1
=
_mm_mul_pd
(
h1_imag
,
q1
);
#ifdef __ELPA_USE_FMA__
x1
=
_mm_add_pd
(
x1
,
_mm_msubadd_pd
(
h1_real
,
q1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x1
=
_mm_add_pd
(
x1
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
q2
);
#ifdef __ELPA_USE_FMA__
x2
=
_mm_add_pd
(
x2
,
_mm_msubadd_pd
(
h1_real
,
q2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
x2
=
_mm_add_pd
(
x2
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
q2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
}
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
0
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[
1
]);
h1_real
=
_mm_xor_pd
(
h1_real
,
sign
);
h1_imag
=
_mm_xor_pd
(
h1_imag
,
sign
);
tmp1
=
_mm_mul_pd
(
h1_imag
,
x1
);
#ifdef __ELPA_USE_FMA__
x1
=
_mm_maddsub_pd
(
h1_real
,
x1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x1
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
x2
);
#ifdef __ELPA_USE_FMA__
x2
=
_mm_maddsub_pd
(
h1_real
,
x2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
)));
#else
x2
=
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
)));
#endif
q1
=
_mm_load_pd
(
&
q_dbl
[
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[
2
]);
q1
=
_mm_add_pd
(
q1
,
x1
);
q2
=
_mm_add_pd
(
q2
,
x2
);
_mm_store_pd
(
&
q_dbl
[
0
],
q1
);
_mm_store_pd
(
&
q_dbl
[
2
],
q2
);
for
(
i
=
1
;
i
<
nb
;
i
++
)
{
h1_real
=
_mm_loaddup_pd
(
&
hh_dbl
[
i
*
2
]);
h1_imag
=
_mm_loaddup_pd
(
&
hh_dbl
[(
i
*
2
)
+
1
]);
q1
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
]);
q2
=
_mm_load_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
]);
tmp1
=
_mm_mul_pd
(
h1_imag
,
x1
);
#ifdef __ELPA_USE_FMA__
q1
=
_mm_add_pd
(
q1
,
_mm_maddsub_pd
(
h1_real
,
x1
,
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q1
=
_mm_add_pd
(
q1
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x1
),
_mm_shuffle_pd
(
tmp1
,
tmp1
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
tmp2
=
_mm_mul_pd
(
h1_imag
,
x2
);
#ifdef __ELPA_USE_FMA__
q2
=
_mm_add_pd
(
q2
,
_mm_maddsub_pd
(
h1_real
,
x2
,
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#else
q2
=
_mm_add_pd
(
q2
,
_mm_addsub_pd
(
_mm_mul_pd
(
h1_real
,
x2
),
_mm_shuffle_pd
(
tmp2
,
tmp2
,
_MM_SHUFFLE2
(
0
,
1
))));
#endif
#define COMPLEXCASE 1
#define DOUBLE_PRECISION 1
#include "../../general/precision_macros.h"
#include "elpa2_kernels_complex_sse_1hv_template.Xc"
#undef DOUBLE_PRECISION
#undef COMPLEXCASE
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
0
],
q1
);
_mm_store_pd
(
&
q_dbl
[(
2
*
i
*
ldq
)
+
2
],
q2
);
}
}
src/elpa2/kernels/elpa2_kernels_complex_sse_1hv_single_precision.c
View file @
863d1b97
...
...
@@ -42,512 +42,14 @@
// any derivatives of ELPA under the same license that we chose for
// the original distribution, the GNU Lesser General Public License.
//
// Author: Andreas Marek, MPCDF, based on the double precision case of A. Heinecke
//
#include "config-f90.h"
#include <complex.h>
#include <x86intrin.h>
#define __forceinline __attribute__((always_inline))
#ifdef HAVE_SSE_INTRINSICS
#undef __AVX__
#endif
//Forward declaration
static
__forceinline
void
hh_trafo_complex_kernel_6_SSE_1hv_single
(
float
complex
*
q
,
float
complex
*
hh
,
int
nb
,
int
ldq
);
static
__forceinline
void
hh_trafo_complex_kernel_4_SSE_1hv_single
(
float
complex
*
q
,
float
complex
*
hh
,
int
nb
,
int
ldq
);
static
__forceinline
void
hh_trafo_complex_kernel_2_SSE_1hv_single
(
float
complex
*
q
,
float
complex
*
hh
,
int
nb
,
int
ldq
);
/*
!f>#ifdef HAVE_SSE_INTRINSICS
!f> interface
!f> subroutine single_hh_trafo_complex_sse_1hv_single(q, hh, pnb, pnq, pldq) &
!f> bind(C, name="single_hh_trafo_complex_sse_1hv_single")
!f> use, intrinsic :: iso_c_binding
!f> integer(kind=c_int) :: pnb, pnq, pldq
!f> ! complex(kind=c_float_complex) :: q(*)