compute_hh_trafo.F90

#if 0
!    This file is part of ELPA.
!
!    The ELPA library was originally created by the ELPA consortium,
!    consisting of the following organizations:
!
!    - Max Planck Computing and Data Facility (MPCDF), formerly known as
!      Rechenzentrum Garching der Max-Planck-Gesellschaft (RZG),
!    - Bergische Universität Wuppertal, Lehrstuhl für angewandte
!      Informatik,
!    - Technische Universität München, Lehrstuhl für Informatik mit
!      Schwerpunkt Wissenschaftliches Rechnen ,
!    - Fritz-Haber-Institut, Berlin, Abt. Theorie,
!    - Max-Plack-Institut für Mathematik in den Naturwissenschaften,
!      Leipzig, Abt. Komplexe Strukutren in Biologie und Kognition,
!      and
!    - IBM Deutschland GmbH
!
!
!    More information can be found here:
!    http://elpa.mpcdf.mpg.de/
!
!    ELPA is free software: you can redistribute it and/or modify
!    it under the terms of the version 3 of the license of the
!    GNU Lesser General Public License as published by the Free
!    Software Foundation.
!
!    ELPA is distributed in the hope that it will be useful,
!    but WITHOUT ANY WARRANTY; without even the implied warranty of
!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
!    GNU Lesser General Public License for more details.
!
!    You should have received a copy of the GNU Lesser General Public License
!    along with ELPA.  If not, see <http://www.gnu.org/licenses/>
!
!    ELPA reflects a substantial effort on the part of the original
!    ELPA consortium, and we ask you to respect the spirit of the
!    license that we chose: i.e., please contribute any changes you
!    may have back to the original ELPA library distribution, and keep
!    any derivatives of ELPA under the same license that we chose for
!    the original distribution, the GNU Lesser General Public License.
!
! This file was written by A. Marek, MPCDF
#endif

subroutine compute_hh_trafo_&
&MATH_DATATYPE&
#ifdef WITH_OPENMP_TRADITIONAL
&_openmp_&
#else
&_&
#endif
&PRECISION &
(obj, useGPU, wantDebug, a, a_dev, stripe_width, a_dim2, stripe_count, max_threads, &
#ifdef WITH_OPENMP_TRADITIONAL
l_nev, &
#endif
a_off, nbw, max_blk_size, bcast_buffer, bcast_buffer_dev, &
hh_tau_dev, kernel_flops, kernel_time, n_times, off, ncols, istripe, &
#ifdef WITH_OPENMP_TRADITIONAL
my_thread, thread_width, kernel, last_stripe_width)
#else
last_stripe_width, kernel)
#endif

  use precision
  use elpa_abstract_impl
  use, intrinsic :: iso_c_binding
#if REALCASE == 1
  use single_hh_trafo_real
#if defined(WITH_REAL_GENERIC_SIMPLE_KERNEL) && !(defined(USE_ASSUMED_SIZE))
  use real_generic_simple_kernel !, only : double_hh_trafo_generic_simple
#endif

#if defined(WITH_REAL_GENERIC_SIMPLE_BLOCK4_KERNEL) && !(defined(USE_ASSUMED_SIZE))
  use real_generic_simple_block4_kernel !, only : double_hh_trafo_generic_simple
#endif

!#if defined(WITH_REAL_GENERIC_SIMPLE_BLOCK6_KERNEL) && !(defined(USE_ASSUMED_SIZE))
!  use real_generic_simple_block6_kernel !, only : double_hh_trafo_generic_simple
!#endif

#if defined(WITH_REAL_GENERIC_KERNEL) && !(defined(USE_ASSUMED_SIZE))
  use real_generic_kernel !, only : double_hh_trafo_generic
#endif

#if defined(WITH_REAL_BGP_KERNEL)
  use real_bgp_kernel !, only : double_hh_trafo_bgp
#endif

#if defined(WITH_REAL_BGQ_KERNEL)
  use real_bgq_kernel !, only : double_hh_trafo_bgq
#endif

#endif /* REALCASE */

#if COMPLEXCASE == 1

#if defined(WITH_COMPLEX_GENERIC_SIMPLE_KERNEL) && !(defined(USE_ASSUMED_SIZE))
  use complex_generic_simple_kernel !, only : single_hh_trafo_complex_generic_simple
#endif
#if defined(WITH_COMPLEX_GENERIC_KERNEL) && !(defined(USE_ASSUMED_SIZE))
  use complex_generic_kernel !, only : single_hh_trafo_complex_generic
#endif

#endif /* COMPLEXCASE */

  !use cuda_c_kernel
  !use cuda_functions
  !use hip_functions
  use gpu_c_kernel
  use elpa_gpu

  use elpa_generated_fortran_interfaces

  implicit none
  class(elpa_abstract_impl_t), intent(inout) :: obj
  logical, intent(in)                        :: useGPU, wantDebug
  real(kind=c_double), intent(inout)         :: kernel_time  ! MPI_WTIME always needs double
  integer(kind=lik)                          :: kernel_flops
  integer(kind=ik), intent(in)               :: nbw, max_blk_size
#if REALCASE == 1
  real(kind=C_DATATYPE_KIND)                 :: bcast_buffer(nbw,max_blk_size)
#endif
#if COMPLEXCASE == 1
  complex(kind=C_DATATYPE_KIND)              :: bcast_buffer(nbw,max_blk_size)
#endif
  integer(kind=ik), intent(in)               :: a_off

  integer(kind=ik), intent(in)               :: stripe_width,a_dim2,stripe_count

  integer(kind=ik), intent(in)               :: max_threads
#ifndef WITH_OPENMP_TRADITIONAL
  integer(kind=ik), intent(in)               :: last_stripe_width
#if REALCASE == 1
!  real(kind=C_DATATYPE_KIND)                :: a(stripe_width,a_dim2,stripe_count)
  real(kind=C_DATATYPE_KIND), pointer        :: a(:,:,:)
#endif
#if COMPLEXCASE == 1
!  complex(kind=C_DATATYPE_KIND)            :: a(stripe_width,a_dim2,stripe_count)
  complex(kind=C_DATATYPE_KIND),pointer     :: a(:,:,:)
#endif

#else /* WITH_OPENMP_TRADITIONAL */
  integer(kind=ik), intent(in)               :: l_nev, thread_width
  integer(kind=ik), intent(in), optional     :: last_stripe_width
#if REALCASE == 1
!  real(kind=C_DATATYPE_KIND)                :: a(stripe_width,a_dim2,stripe_count,max_threads)
  real(kind=C_DATATYPE_KIND), pointer        :: a(:,:,:,:)
#endif
#if COMPLEXCASE == 1
!  complex(kind=C_DATATYPE_KIND)            :: a(stripe_width,a_dim2,stripe_count,max_threads)
  complex(kind=C_DATATYPE_KIND),pointer     :: a(:,:,:,:)
#endif

#endif /* WITH_OPENMP_TRADITIONAL */

  integer(kind=ik), intent(in)               :: kernel

  integer(kind=c_intptr_t)                   :: a_dev
  integer(kind=c_intptr_t)                   :: bcast_buffer_dev
  integer(kind=c_intptr_t)                   :: hh_tau_dev
  integer(kind=c_intptr_t)                   :: dev_offset, dev_offset_1, dev_offset_2

  ! Private variables in OMP regions (my_thread) should better be in the argument list!
  integer(kind=ik)                           :: off, ncols, istripe
#ifdef WITH_OPENMP_TRADITIONAL
  integer(kind=ik)                           :: my_thread, noff
#endif
  integer(kind=ik)                           :: j, nl, jj, jjj, n_times
#if REALCASE == 1
  real(kind=C_DATATYPE_KIND)                 :: w(nbw,6)
#endif
#if COMPLEXCASE == 1
  complex(kind=C_DATATYPE_KIND)              :: w(nbw,2)
#endif
  real(kind=c_double)                        :: ttt ! MPI_WTIME always needs double

  integer(kind=c_intptr_t), parameter        :: size_of_datatype = size_of_&
                                                                 &PRECISION&
                                                                 &_&
                                                                 &MATH_DATATYPE


  j = -99

  if (wantDebug) then
    if (useGPU .and. &
#if REALCASE == 1
      ( kernel .ne. ELPA_2STAGE_REAL_NVIDIA_GPU)) then
#endif
#if COMPLEXCASE == 1
      ( kernel .ne. ELPA_2STAGE_COMPLEX_NVIDIA_GPU)) then
#endif
      print *,"ERROR: useGPU is set in conpute_hh_trafo but not GPU kernel!"
      stop
    endif
  endif

#if REALCASE == 1
  if (kernel .eq. ELPA_2STAGE_REAL_NVIDIA_GPU) then
#endif
#if COMPLEXCASE == 1
  if (kernel .eq. ELPA_2STAGE_COMPLEX_NVIDIA_GPU) then
#endif
    ! ncols - indicates the number of HH reflectors to apply; at least 1 must be available
    if (ncols < 1) then
      if (wantDebug) then
        print *, "Returning early from compute_hh_trafo"
      endif
      return
    endif
  endif

  if (wantDebug) call obj%timer%start("compute_hh_trafo_&
  &MATH_DATATYPE&
#ifdef WITH_OPENMP_TRADITIONAL
  &_openmp" // &
#else
  &" // &
#endif
  &PRECISION_SUFFIX &
  )


#ifdef WITH_OPENMP_TRADITIONAL
  if (my_thread==1) then ! in the calling routine threads go form 1 .. max_threads
#endif
    ttt = mpi_wtime()
#ifdef WITH_OPENMP_TRADITIONAL
  endif
#endif


#ifndef WITH_OPENMP_TRADITIONAL
  nl = merge(stripe_width, last_stripe_width, istripe<stripe_count)
#else /* WITH_OPENMP_TRADITIONAL */

  if (present(last_stripe_width)) then
    nl = merge(stripe_width, last_stripe_width, istripe<stripe_count)
  else
    if (istripe<stripe_count) then
      nl = stripe_width
    else
      noff = (my_thread-1)*thread_width + (istripe-1)*stripe_width
      nl = min(my_thread*thread_width-noff, l_nev-noff)
      if (nl<=0) then
        if (wantDebug) call obj%timer%stop("compute_hh_trafo_&
        &MATH_DATATYPE&
#ifdef WITH_OPENMP_TRADITIONAL
        &_openmp" // &
#else
        &" // &
#endif
        &PRECISION_SUFFIX &
        )

        return
      endif
    endif
  endif
#endif /* not WITH_OPENMP_TRADITIONAL */

#if REALCASE == 1
! GPU kernel real
  if (kernel .eq. ELPA_2STAGE_REAL_NVIDIA_GPU) then
#endif
#if COMPLEXCASE == 1
! GPU kernel complex
  if (kernel .eq. ELPA_2STAGE_COMPLEX_NVIDIA_GPU) then
#endif
    if (wantDebug) then
      call obj%timer%start("compute_hh_trafo: GPU")
    endif

    dev_offset = ((a_off+off)*stripe_width+(istripe-1)*stripe_width*a_dim2)*size_of_datatype

    dev_offset_1 = off*nbw*size_of_datatype

    dev_offset_2 = off*size_of_datatype

    call launch_compute_hh_trafo_gpu_kernel_&
         &MATH_DATATYPE&
         &_&
         &PRECISION&
         &(a_dev + dev_offset, bcast_buffer_dev + dev_offset_1, &
         hh_tau_dev + dev_offset_2, nl, nbw,stripe_width, ncols)

    if (wantDebug) then
      call obj%timer%stop("compute_hh_trafo: GPU")
    endif

  else ! not CUDA kernel

    if (wantDebug) then
      call obj%timer%start("compute_hh_trafo: CPU")
    endif
#if REALCASE == 1
#ifndef WITH_FIXED_REAL_KERNEL
    if (kernel .eq. ELPA_2STAGE_REAL_AVX_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_AVX2_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_AVX512_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_SSE_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_SPARC64_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_VSX_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_SVE128_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_SVE256_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_SVE512_BLOCK2 .or. &
        kernel .eq. ELPA_2STAGE_REAL_GENERIC    .or. &
        kernel .eq. ELPA_2STAGE_REAL_GENERIC_SIMPLE .or. &
        kernel .eq. ELPA_2STAGE_REAL_SSE_ASSEMBLY .or. &
        kernel .eq. ELPA_2STAGE_REAL_BGP .or.        &
        kernel .eq. ELPA_2STAGE_REAL_BGQ) then
#endif /* not WITH_FIXED_REAL_KERNEL */

#endif /* REALCASE */

      !FORTRAN CODE / X86 INRINISIC CODE / BG ASSEMBLER USING 2 HOUSEHOLDER VECTORS
#if REALCASE == 1
      ! generic kernel real case
#if defined(WITH_REAL_GENERIC_KERNEL)
#ifndef WITH_FIXED_REAL_KERNEL
      if (kernel .eq. ELPA_2STAGE_REAL_GENERIC) then
#endif /* not WITH_FIXED_REAL_KERNEL */

        do j = ncols, 2, -2
          w(:,1) = bcast_buffer(1:nbw,j+off)
          w(:,2) = bcast_buffer(1:nbw,j+off-1)

#ifdef WITH_OPENMP_TRADITIONAL

#ifdef USE_ASSUMED_SIZE
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_generic_&
          &PRECISION&
          & (a(1,j+off+a_off-1,istripe,my_thread), w, nbw, nl, stripe_width, nbw)
#else
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_generic_&
          &PRECISION&
          & (a(1:stripe_width,j+off+a_off-1:j+off+a_off+nbw-1, istripe,my_thread), w(1:nbw,1:6), &
          nbw, nl, stripe_width, nbw)
#endif

#else /* WITH_OPENMP_TRADITIONAL */

#ifdef USE_ASSUMED_SIZE
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_generic_&
          &PRECISION&
          & (a(1,j+off+a_off-1,istripe),w, nbw, nl, stripe_width, nbw)
#else
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_generic_&
          &PRECISION&
          & (a(1:stripe_width,j+off+a_off-1:j+off+a_off+nbw-1,istripe),w(1:nbw,1:6), nbw, nl, stripe_width, nbw)
#endif
#endif /* WITH_OPENMP_TRADITIONAL */

        enddo

#ifndef WITH_FIXED_REAL_KERNEL
      endif
#endif /* not WITH_FIXED_REAL_KERNEL */
#endif /* WITH_REAL_GENERIC_KERNEL */

#endif /* REALCASE == 1 */

#if COMPLEXCASE == 1
      ! generic kernel complex case
#if defined(WITH_COMPLEX_GENERIC_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if (kernel .eq. ELPA_2STAGE_COMPLEX_GENERIC .or. &
          kernel .eq. ELPA_2STAGE_COMPLEX_BGP .or. &
          kernel .eq. ELPA_2STAGE_COMPLEX_BGQ ) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
          ttt = mpi_wtime()
          do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
#ifdef USE_ASSUMED_SIZE

            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_&
                 &PRECISION&
                 & (a(1,j+off+a_off,istripe,my_thread), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_&
                 &PRECISION&
                 & (a(1:stripe_width,j+off+a_off:j+off+a_off+nbw-1,istripe,my_thread), &
                 bcast_buffer(1:nbw,j+off), nbw, nl, stripe_width)
#endif

#else /* WITH_OPENMP_TRADITIONAL */

#ifdef USE_ASSUMED_SIZE
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_&
                 &PRECISION&
                 & (a(1,j+off+a_off,istripe), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_&
                 &PRECISION&
                 & (a(1:stripe_width,j+off+a_off:j+off+a_off+nbw-1,istripe), bcast_buffer(1:nbw,j+off), &
                 nbw, nl, stripe_width)
#endif
#endif /* WITH_OPENMP_TRADITIONAL */

          enddo
#ifndef WITH_FIXED_COMPLEX_KERNEL
        endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_GENERIC .or. kernel .eq. ELPA_2STAGE_COMPLEX_BGP .or. kernel .eq. ELPA_2STAGE_COMPLEX_BGQ )
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_GENERIC_KERNEL */

#endif /* COMPLEXCASE */

#if REALCASE == 1
        ! generic simple real kernel
#if defined(WITH_REAL_GENERIC_SIMPLE_KERNEL)
#ifndef WITH_FIXED_REAL_KERNEL
        if (kernel .eq. ELPA_2STAGE_REAL_GENERIC_SIMPLE) then
#endif /* not WITH_FIXED_REAL_KERNEL */
          do j = ncols, 2, -2
            w(:,1) = bcast_buffer(1:nbw,j+off)
            w(:,2) = bcast_buffer(1:nbw,j+off-1)
#ifdef WITH_OPENMP_TRADITIONAL

#ifdef USE_ASSUMED_SIZE
            call double_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1,j+off+a_off-1,istripe,my_thread), w, nbw, nl, stripe_width, nbw)
#else
            call double_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1:stripe_width,j+off+a_off-1:j+off+a_off-1+nbw,istripe,my_thread), w, nbw, nl, stripe_width, nbw)

#endif

#else /* WITH_OPENMP_TRADITIONAL */

#ifdef USE_ASSUMED_SIZE
            call double_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1,j+off+a_off-1,istripe), w, nbw, nl, stripe_width, nbw)
#else
            call double_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1:stripe_width,j+off+a_off-1:j+off+a_off-1+nbw,istripe), w, nbw, nl, stripe_width, nbw)
#endif

#endif /* WITH_OPENMP_TRADITIONAL */

          enddo
#ifndef WITH_FIXED_REAL_KERNEL
        endif
#endif /* not WITH_FIXED_REAL_KERNEL */
#endif /* WITH_REAL_GENERIC_SIMPLE_KERNEL */

#endif /* REALCASE */

#if COMPLEXCASE == 1
        ! generic simple complex case

#if defined(WITH_COMPLEX_GENERIC_SIMPLE_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
        if (kernel .eq. ELPA_2STAGE_COMPLEX_GENERIC_SIMPLE) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
          ttt = mpi_wtime()
          do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
#ifdef USE_ASSUMED_SIZE
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1,j+off+a_off,istripe,my_thread), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1:stripe_width, j+off+a_off:j+off+a_off+nbw-1,istripe,my_thread), bcast_buffer(1:nbw,j+off), &
                 nbw, nl, stripe_width)
#endif

#else /* WITH_OPENMP_TRADITIONAL */

#ifdef USE_ASSUMED_SIZE
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1,j+off+a_off,istripe), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
            call single_hh_trafo_&
                 &MATH_DATATYPE&
                 &_generic_simple_&
                 &PRECISION&
                 & (a(1:stripe_width,j+off+a_off:j+off+a_off+nbw-1,istripe), bcast_buffer(1:nbw,j+off), &
                 nbw, nl, stripe_width)
#endif

#endif /* WITH_OPENMP_TRADITIONAL */
          enddo
#ifndef WITH_FIXED_COMPLEX_KERNEL
        endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_GENERIC_SIMPLE)
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_GENERIC_SIMPLE_KERNEL */

#endif /* COMPLEXCASE */

#if REALCASE == 1
        ! sse assembly kernel real case
#if defined(WITH_REAL_SSE_ASSEMBLY_KERNEL)
#ifndef WITH_FIXED_REAL_KERNEL
        if (kernel .eq. ELPA_2STAGE_REAL_SSE_ASSEMBLY) then

#endif /* not WITH_FIXED_REAL_KERNEL */
          do j = ncols, 2, -2
            w(:,1) = bcast_buffer(1:nbw,j+off)
            w(:,2) = bcast_buffer(1:nbw,j+off-1)
#ifdef WITH_OPENMP_TRADITIONAL
            call double_hh_trafo_&
            &MATH_DATATYPE&
            &_&
            &PRECISION&
            &_sse_assembly&
            & (c_loc(a(1,j+off+a_off-1,istripe,my_thread)), w, nbw, nl, stripe_width, nbw)
#else
            call double_hh_trafo_&
            &MATH_DATATYPE&
            &_&
            &PRECISION&
            &_sse_assembly&
            & (c_loc(a(1,j+off+a_off-1,istripe)), w, nbw, nl, stripe_width, nbw)
#endif
          enddo
#ifndef WITH_FIXED_REAL_KERNEL
        endif
#endif /* not WITH_FIXED_REAL_KERNEL */
#endif /* WITH_REAL_SSE_ASSEMBLY_KERNEL */

#endif /* REALCASE */

#if COMPLEXCASE == 1

        ! sse assembly kernel complex case
#if defined(WITH_COMPLEX_SSE_ASSEMBLY_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
        if (kernel .eq. ELPA_2STAGE_COMPLEX_SSE_ASSEMBLY) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
          ttt = mpi_wtime()
          do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
            call single_hh_trafo_&
            &MATH_DATATYPE&
            &_&
            &PRECISION&
            &_sse_assembly&
            & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
            call single_hh_trafo_&
            &MATH_DATATYPE&
            &_&
            &PRECISION&
            &_sse_assembly&
            & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
          enddo
#ifndef WITH_FIXED_COMPLEX_KERNEL
        endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_SSE)
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_SSE_ASSEMBLY_KERNEL */
#endif /* COMPLEXCASE */

#if REALCASE == 1
        ! no sse, vsx, sparc64 sve block1 real kernel
#endif

#if COMPLEXCASE == 1

        ! sparc64 block1 complex kernel
#if defined(WITH_COMPLEX_SPARC64_BLOCK1_KERNEL)
!#ifndef WITH_FIXED_COMPLEX_KERNEL
!        if (kernel .eq. ELPA_2STAGE_COMPLEX_SPARC64_BLOCK1) then
!#endif /* not WITH_FIXED_COMPLEX_KERNEL */
!
!#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SPARC64_BLOCK2_KERNEL))
!        ttt = mpi_wtime()
!        do j = ncols, 1, -1
!#ifdef WITH_OPENMP_TRADITIONAL
!          call single_hh_trafo_&
!          &MATH_DATATYPE&
!          &_sparc64_1hv_&
!          &PRECISION&
!          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
!#else
!          call single_hh_trafo_&
!          &MATH_DATATYPE&
!          &_sparc64_1hv_&
!          &PRECISION&
!          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
!#endif
!        enddo
!#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SPARC64_BLOCK2_KERNEL)) */
!
!#ifndef WITH_FIXED_COMPLEX_KERNEL
!      endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_SPARC64_BLOCK1)
!#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_SPARC64_BLOCK1_KERNEL */

#endif /* COMPLEXCASE */


#if COMPLEXCASE == 1

      ! vsx block1 complex kernel
#if defined(WITH_COMPLEX_VSX_BLOCK1_KERNEL)
!#ifndef WITH_FIXED_COMPLEX_KERNEL
!      if (kernel .eq. ELPA_2STAGE_COMPLEX_VSX_BLOCK1) then
!#endif /* not WITH_FIXED_COMPLEX_KERNEL */
!
!#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_VSX_BLOCK2_KERNEL))
!        ttt = mpi_wtime()
!        do j = ncols, 1, -1
!#ifdef WITH_OPENMP_TRADITIONAL
!          call single_hh_trafo_&
!          &MATH_DATATYPE&
!          &_vsx_1hv_&
!          &PRECISION&
!          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
!#else
!          call single_hh_trafo_&
!          &MATH_DATATYPE&
!          &_vsx_1hv_&
!          &PRECISION&
!          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
!#endif
!        enddo
!#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_VSX_BLOCK2_KERNEL)) */
!
!#ifndef WITH_FIXED_COMPLEX_KERNEL
!      endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_VSX_BLOCK1)
!#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_VSX_BLOCK1_KERNEL */

#endif /* COMPLEXCASE */


#if COMPLEXCASE == 1

      ! sse block1 complex kernel
#if defined(WITH_COMPLEX_SSE_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if (kernel .eq. ELPA_2STAGE_COMPLEX_SSE_BLOCK1) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SSE_BLOCK2_KERNEL))
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sse_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sse_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SSE_BLOCK2_KERNEL)) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_SSE_BLOCK1)
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_SSE_BLOCK1_KERNEL */

      ! neon_arch64 block1 complex kernel
#if defined(WITH_COMPLEX_NEON_ARCH64_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if (kernel .eq. ELPA_2STAGE_COMPLEX_NEON_ARCH64_BLOCK1) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_NEON_ARCH64_BLOCK2_KERNEL))
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_neon_arch64_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_neon_arch64_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_NEON_ARCH64_BLOCK2_KERNEL)) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_NEON_ARCH64_BLOCK1)
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_NEON_ARCH64_BLOCK1_KERNEL */

      ! sve128 block1 complex kernel
#if defined(WITH_COMPLEX_SVE128_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if (kernel .eq. ELPA_2STAGE_COMPLEX_SVE128_BLOCK1) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SVE128_BLOCK2_KERNEL))
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sve128_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sve128_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SVE128_BLOCK2_KERNEL)) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! (kernel .eq. ELPA_2STAGE_COMPLEX_SVE128_BLOCK1)
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_SVE128_BLOCK1_KERNEL */

#endif /* COMPLEXCASE */

#if REALCASE == 1
      !no avx block1 real kernel
#endif /* REALCASE */

#if COMPLEXCASE == 1

      ! avx block1 complex kernel
#if defined(WITH_COMPLEX_AVX_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if ((kernel .eq. ELPA_2STAGE_COMPLEX_AVX_BLOCK1)) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_AVX_BLOCK2_KERNEL) )
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_avx_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_avx_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_AVX_BLOCK2_KERNEL)) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! ((kernel .eq. ELPA_2STAGE_COMPLEX_AVX_BLOCK1) )
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_AVX_BLOCK1_KERNEL */

#if defined(WITH_COMPLEX_AVX2_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if ((kernel .eq. ELPA_2STAGE_COMPLEX_AVX2_BLOCK1)) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_AVX2_BLOCK2_KERNEL))
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_avx2_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_avx2_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_AVX2_BLOCK2_KERNEL)) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! ((kernel .eq. ELPA_2STAGE_COMPLEX_AVX2_BLOCK1))
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_AVX2_BLOCK1_KERNEL */

#if defined(WITH_COMPLEX_SVE256_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if ((kernel .eq. ELPA_2STAGE_COMPLEX_SVE256_BLOCK1)) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SVE256_BLOCK2_KERNEL))
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sve256_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sve256_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SVE256_BLOCK2_KERNEL)) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! ((kernel .eq. ELPA_2STAGE_COMPLEX_SVE256_BLOCK1))
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_SVE256_BLOCK1_KERNEL */

#endif /* COMPLEXCASE */

#if REALCASE == 1
      ! no avx512 block1 real kernel
      ! no sve512 block1 real kernel
#endif /* REALCASE */

#if COMPLEXCASE == 1

      ! avx512 block1 complex kernel
#if defined(WITH_COMPLEX_AVX512_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if ((kernel .eq. ELPA_2STAGE_COMPLEX_AVX512_BLOCK1)) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_AVX512_BLOCK2_KERNEL) )
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_avx512_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_avx512_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_AVX512_BLOCK2_KERNEL) ) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! ((kernel .eq. ELPA_2STAGE_COMPLEX_AVX512_BLOCK1))
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_AVX512_BLOCK1_KERNEL  */

      ! sve512 block1 complex kernel
#if defined(WITH_COMPLEX_SVE512_BLOCK1_KERNEL)
#ifndef WITH_FIXED_COMPLEX_KERNEL
      if ((kernel .eq. ELPA_2STAGE_COMPLEX_SVE512_BLOCK1)) then
#endif /* not WITH_FIXED_COMPLEX_KERNEL */

#if (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SVE512_BLOCK2_KERNEL) )
        ttt = mpi_wtime()
        do j = ncols, 1, -1
#ifdef WITH_OPENMP_TRADITIONAL
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sve512_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe,my_thread)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#else
          call single_hh_trafo_&
          &MATH_DATATYPE&
          &_sve512_1hv_&
          &PRECISION&
          & (c_loc(a(1,j+off+a_off,istripe)), bcast_buffer(1,j+off),nbw,nl,stripe_width)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_COMPLEX_KERNEL)) || (defined(WITH_FIXED_COMPLEX_KERNEL) && !defined(WITH_COMPLEX_SVE512_BLOCK2_KERNEL) ) */

#ifndef WITH_FIXED_COMPLEX_KERNEL
      endif ! ((kernel .eq. ELPA_2STAGE_COMPLEX_SVE512_BLOCK1))
#endif /* not WITH_FIXED_COMPLEX_KERNEL */
#endif /* WITH_COMPLEX_SVE512_BLOCK1_KERNEL  */

#endif /* COMPLEXCASE */

#if REALCASE == 1
      ! implementation of sparc64 block 2 real case
#if defined(WITH_REAL_SPARC64_BLOCK2_KERNEL)

#ifndef WITH_FIXED_REAL_KERNEL
      if (kernel .eq. ELPA_2STAGE_REAL_SPARC64_BLOCK2) then

#endif /* not WITH_FIXED_REAL_KERNEL */

#if (!defined(WITH_FIXED_REAL_KERNEL)) || (defined(WITH_FIXED_REAL_KERNEL) && !defined(WITH_REAL_SPARC64_BLOCK6_KERNEL) && !defined(WITH_REAL_SPARC64_BLOCK4_KERNEL))
        do j = ncols, 2, -2
          w(:,1) = bcast_buffer(1:nbw,j+off)
          w(:,2) = bcast_buffer(1:nbw,j+off-1)
#ifdef WITH_OPENMP_TRADITIONAL
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_sparc64_2hv_&
          &PRECISION &
          & (c_loc(a(1,j+off+a_off-1,istripe,my_thread)), w, nbw, nl, stripe_width, nbw)
#else
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_sparc64_2hv_&
          &PRECISION &
          & (c_loc(a(1,j+off+a_off-1,istripe)), w, nbw, nl, stripe_width, nbw)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_REAL_KERNEL)) || (defined(WITH_FIXED_REAL_KERNEL) && !defined(WITH_REAL_SPARC64_BLOCK6_KERNEL) && !defined(WITH_REAL_SPARC64_BLOCK4_KERNEL)) */

#ifndef WITH_FIXED_REAL_KERNEL
      endif
#endif /* not WITH_FIXED_REAL_KERNEL */
#endif /* WITH_REAL_SPARC64_BLOCK2_KERNEL */

#endif /* REALCASE == 1 */

#if REALCASE == 1
      ! implementation of neon_arch64 block 2 real case
#if defined(WITH_REAL_NEON_ARCH64_BLOCK2_KERNEL)

#ifndef WITH_FIXED_REAL_KERNEL
      if (kernel .eq. ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK2) then

#endif /* not WITH_FIXED_REAL_KERNEL */

#if (!defined(WITH_FIXED_REAL_KERNEL)) || (defined(WITH_FIXED_REAL_KERNEL) && !defined(WITH_REAL_NEON_ARCH64_BLOCK6_KERNEL) && !defined(WITH_REAL_NEON_ARCH64_BLOCK4_KERNEL))
        do j = ncols, 2, -2
          w(:,1) = bcast_buffer(1:nbw,j+off)
          w(:,2) = bcast_buffer(1:nbw,j+off-1)
#ifdef WITH_OPENMP_TRADITIONAL
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_neon_arch64_2hv_&
          &PRECISION &
          & (c_loc(a(1,j+off+a_off-1,istripe,my_thread)), w, nbw, nl, stripe_width, nbw)
#else
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_neon_arch64_2hv_&
          &PRECISION &
          & (c_loc(a(1,j+off+a_off-1,istripe)), w, nbw, nl, stripe_width, nbw)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_REAL_KERNEL)) || (defined(WITH_FIXED_REAL_KERNEL) && !defined(WITH_REAL_NEON_ARCH64_BLOCK6_KERNEL) && !defined(WITH_REAL_NEON_ARCH64_BLOCK4_KERNEL)) */

#ifndef WITH_FIXED_REAL_KERNEL
      endif
#endif /* not WITH_FIXED_REAL_KERNEL */
#endif /* WITH_REAL_NEON_ARCH64_BLOCK2_KERNEL */

      ! implementation of neon_arch64 block 2 real case
#if defined(WITH_REAL_SVE128_BLOCK2_KERNEL)

#ifndef WITH_FIXED_REAL_KERNEL
      if (kernel .eq. ELPA_2STAGE_REAL_SVE128_BLOCK2) then

#endif /* not WITH_FIXED_REAL_KERNEL */

#if (!defined(WITH_FIXED_REAL_KERNEL)) || (defined(WITH_FIXED_REAL_KERNEL) && !defined(WITH_REAL_SVE128_BLOCK6_KERNEL) && !defined(WITH_REAL_SVE128_BLOCK4_KERNEL))
        do j = ncols, 2, -2
          w(:,1) = bcast_buffer(1:nbw,j+off)
          w(:,2) = bcast_buffer(1:nbw,j+off-1)
#ifdef WITH_OPENMP_TRADITIONAL
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_sve128_2hv_&
          &PRECISION &
          & (c_loc(a(1,j+off+a_off-1,istripe,my_thread)), w, nbw, nl, stripe_width, nbw)
#else
          call double_hh_trafo_&
          &MATH_DATATYPE&
          &_sve128_2hv_&
          &PRECISION &
          & (c_loc(a(1,j+off+a_off-1,istripe)), w, nbw, nl, stripe_width, nbw)
#endif
        enddo
#endif /* (!defined(WITH_FIXED_REAL_KERNEL)) || (defined(WITH_FIXED_REAL_KERNEL) && !defined(WITH_REAL_SVE128_BLOCK6_KERNEL) && !defined(WITH_REAL_SVE128_BLOCK4_KERNEL)) */

#ifndef WITH_FIXED_REAL_KERNEL
      endif
#endif /* not WITH_FIXED_REAL_KERNEL */
#endif /* WITH_REAL_SVE128_BLOCK2_KERNEL */