!
! Copyright (C) 2021, MaX CoE
! Distributed under the MIT License
! (license terms are at http://opensource.org/licenses/MIT).
!
!--
!
! Testing: Linear Algebra interfaces and wrappers
!
#include<devxlib_macros.h>
#include<devxlib_defs.h>
!
program test_linalg
  !
  use iso_fortran_env
  use iso_c_binding
  use devxlib_linalg
  use devxlib_memcpy
  use devxlib_mapping
  use devxlib_async
  use myprof
  !
#if defined __DXL_CUDAF
  use cudafor
#endif
  !
  implicit none

  interface random_number_cmplx
     procedure :: random_number_cmplx_c1, random_number_cmplx_c2, &
                  random_number_cmplx_z1, random_number_cmplx_z2
  end interface

  complex(real32), parameter :: ci_sp=cmplx(0.0,1.0,kind=real32)
  complex(real64), parameter :: ci_dp=cmplx(0.0,1.0,kind=real64)
  integer(int64) :: cnt, count_max
  real(real64), parameter :: thr_SP=1.0d-6
  real(real64), parameter :: thr_DP=1.0d-12
  !
  ! naming scheme: v  vector
  !                M  matrix
  !                s d c z   real/comple precision
  !                _h, _d    host/device
  !
  real(real32), allocatable :: s_vx_h1(:), s_vy_h1(:)
  real(real32), allocatable :: s_vx_h2(:), s_vy_h2(:)
  real(real32), allocatable DEV_ATTR :: s_vx_d1(:), s_vy_d1(:)
  real(real64), allocatable :: d_vx_h1(:), d_vy_h1(:)
  real(real64), allocatable :: d_vx_h2(:), d_vy_h2(:)
  real(real64), allocatable DEV_ATTR :: d_vx_d1(:), d_vy_d1(:)
  complex(real32), allocatable :: c_vx_h1(:), c_vy_h1(:)
  complex(real32), allocatable :: c_vx_h2(:), c_vy_h2(:)
  complex(real32), allocatable DEV_ATTR :: c_vx_d1(:), c_vy_d1(:)
  complex(real64), allocatable :: z_vx_h1(:), z_vy_h1(:)
  complex(real64), allocatable :: z_vx_h2(:), z_vy_h2(:)
  complex(real64), allocatable DEV_ATTR :: z_vx_d1(:), z_vy_d1(:)
  !
  real(real32), allocatable :: s_mA_h1(:,:)
  real(real32), allocatable :: s_mB_h1(:,:)
  real(real32), allocatable :: s_mC_h1(:,:)
  real(real32), allocatable :: s_mC_h2(:,:)
  real(real32), allocatable DEV_ATTR :: s_mA_d1(:,:)
  real(real32), allocatable DEV_ATTR :: s_mB_d1(:,:)
  real(real32), allocatable DEV_ATTR :: s_mC_d1(:,:)
  real(real64), allocatable :: d_mA_h1(:,:)
  real(real64), allocatable :: d_mB_h1(:,:)
  real(real64), allocatable :: d_mC_h1(:,:)
  real(real64), allocatable :: d_mC_h2(:,:)
  real(real64), allocatable DEV_ATTR :: d_mA_d1(:,:)
  real(real64), allocatable DEV_ATTR :: d_mB_d1(:,:)
  real(real64), allocatable DEV_ATTR :: d_mC_d1(:,:)
  complex(real32), allocatable :: c_mA_h1(:,:)
  complex(real32), allocatable :: c_mB_h1(:,:)
  complex(real32), allocatable :: c_mC_h1(:,:)
  complex(real32), allocatable :: c_mC_h2(:,:)
  complex(real32), allocatable DEV_ATTR :: c_mA_d1(:,:)
  complex(real32), allocatable DEV_ATTR :: c_mB_d1(:,:)
  complex(real32), allocatable DEV_ATTR :: c_mC_d1(:,:)
  complex(real64), allocatable :: z_mA_h1(:,:)
  complex(real64), allocatable :: z_mB_h1(:,:)
  complex(real64), allocatable :: z_mC_h1(:,:)
  complex(real64), allocatable :: z_mC_h2(:,:)
  complex(real64), allocatable DEV_ATTR :: z_mA_d1(:,:)
  complex(real64), allocatable DEV_ATTR :: z_mB_d1(:,:)
  complex(real64), allocatable DEV_ATTR :: z_mC_d1(:,:)
  !
  real(real32) :: s_alpha, s_beta
  real(real64) :: d_alpha, d_beta
  complex(real32) :: c_alpha, c_beta
  complex(real64) :: z_alpha, z_beta
  real(real32) :: s_scal_h1, s_scal_h2
  real(real64) :: d_scal_h1, d_scal_h2
  complex(real32) :: c_scal_h1, c_scal_h2, c_scal_h3, c_scal_h4
  complex(real64) :: z_scal_h1, z_scal_h2, z_scal_h3, z_scal_h4
  !
  real(real64) :: t0, t1, count_rate
  real(real64) :: norm
  integer :: i,j,k,ierr
  integer :: ndim(4)
  integer :: nfail, npass
  !
  integer(kind=dxl_async_kind) :: default_async_id
  character(256) :: arg, str

!
!============================
! get dims
!============================
!
  ! defaults
  ndim(1:4)=[100,200,300,400]
  !
  i=0
  do
    call get_command_argument(i, arg)
    if (len_trim(arg) == 0) exit
    !
    select case (trim(arg))
    case("-h","--help")
      write(output_unit,"(a)") "Usage: "
      write(output_unit,"(a)") "   ./test_linalg.x [--dims <vals>] "
      stop
    end select
    !
    i = i+1
    call get_command_argument(i, str)
    if (len_trim(str) == 0) exit
    !
    select case (trim(arg))
    case("-dims","--dims")
      read(str,*,iostat=ierr) ndim(:)
      if (ierr/=0) STOP "reading cmd-line args: dims"
    end select
  enddo
  !
  write(output_unit,"(/,a,/)") "Running test with params: "
  write(output_unit,"(3x,a,10i5)") "  ndim: ", ndim(:)
  write(output_unit,"()")
  !
  npass=0
  nfail=0

  !
  ! async_id (streams/queues)
  !
#if defined __DXL_CUDAF || defined __DXL_OPENACC
  call devxlib_async_create(default_async_id,default_async=.true.)
#endif
!
!============================
! check linalg
!============================
!
  write(output_unit,"(/,3x,a)") "Initialize LinAlg ..."
  call dev_linalg_setup()
  write(output_unit,"()")

  write(output_unit,"(/,3x,a)") "Initialize profiling ..."
  call init_prof()
  write(output_unit,"()")

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xDOT (S) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( s_vx_h1(ndim(1)) )
  allocate( s_vy_h1(ndim(1)) )
  allocate( s_vx_d1(ndim(1)) )
  allocate( s_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(s_vx_d1)
  call devxlib_map(s_vy_d1)
#endif
  !
  ! init
  call random_number( s_vx_h1 )
  call random_number( s_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(s_vx_d1, s_vx_h1)
  call devxlib_memcpy_h2d(s_vy_d1, s_vy_h1)
  !
  ! host LinAlg
  s_scal_h1 = devxlib_xDOT( ndim(1), s_vx_h1, 1, s_vy_h1, 1)
  !
  ! dev LinAlg
  s_scal_h2 = devxlib_xDOT_gpu( ndim(1), s_vx_d1, 1, s_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  norm=10.0d0*abs(s_scal_h1)/sqrt(real(ndim(1)))
  !
  if ( abs(s_scal_h2-s_scal_h1)> thr_SP*norm ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( s_vx_h1 )
  deallocate( s_vy_h1 )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(s_vx_d1)
  call devxlib_unmap(s_vy_d1)
#endif
  deallocate( s_vx_d1 )
  deallocate( s_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xDOT (D) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( d_vx_h1(ndim(1)) )
  allocate( d_vy_h1(ndim(1)) )
  allocate( d_vx_d1(ndim(1)) )
  allocate( d_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(d_vx_d1)
  call devxlib_map(d_vy_d1)
#endif
  !
  ! init
  call random_number( d_vx_h1 )
  call random_number( d_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(d_vx_d1, d_vx_h1)
  call devxlib_memcpy_h2d(d_vy_d1, d_vy_h1)
  !
  ! host LinAlg
  call start_prof("xDOT_cpu")
  d_scal_h1 = devxlib_xDOT( ndim(1), d_vx_h1, 1, d_vy_h1, 1)
  call stop_prof("xDOT_cpu")
  !
  ! dev LinAlg
  call start_prof("xDOT_gpu")
  d_scal_h2 = devxlib_xDOT_gpu( ndim(1), d_vx_d1, 1, d_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  call stop_prof("xDOT_gpu")
  !
  norm=10.0d0*abs(d_scal_h1)/sqrt(real(ndim(1)))
  !
  if ( abs(d_scal_h2-d_scal_h1)> thr_DP*norm ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( d_vx_h1 )
  deallocate( d_vy_h1 )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(d_vx_d1)
  call devxlib_unmap(d_vy_d1)
#endif
  deallocate( d_vx_d1 )
  deallocate( d_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xDOTU/C (C) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( c_vx_h1(ndim(1)) )
  allocate( c_vy_h1(ndim(1)) )
  allocate( c_vx_d1(ndim(1)) )
  allocate( c_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(c_vx_d1)
  call devxlib_map(c_vy_d1)
#endif
  !
  ! init
  call random_number_cmplx( c_vx_h1 )
  call random_number_cmplx( c_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(c_vx_d1, c_vx_h1)
  call devxlib_memcpy_h2d(c_vy_d1, c_vy_h1)
  !
  ! host LinAlg
  c_scal_h1 = devxlib_xDOTU( ndim(1), c_vx_h1, 1, c_vy_h1, 1)
  c_scal_h3 = devxlib_xDOTC( ndim(1), c_vx_h1, 1, c_vy_h1, 1)
  !
  ! dev LinAlg
  c_scal_h2 = devxlib_xDOTU_gpu( ndim(1), c_vx_d1, 1, c_vy_d1, 1)
  c_scal_h4 = devxlib_xDOTC_gpu( ndim(1), c_vx_d1, 1, c_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  norm=10.0d0*abs(c_scal_h1)/sqrt(real(ndim(1)))
  !
  if ( abs(c_scal_h2-c_scal_h1)> thr_SP*norm .and. &
       abs(c_scal_h4-c_scal_h3)> thr_SP*norm ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( c_vx_h1 )
  deallocate( c_vy_h1 )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(c_vx_d1)
  call devxlib_unmap(c_vy_d1)
#endif
  deallocate( c_vx_d1 )
  deallocate( c_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xDOTU/C (Z) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( z_vx_h1(ndim(1)) )
  allocate( z_vy_h1(ndim(1)) )
  allocate( z_vx_d1(ndim(1)) )
  allocate( z_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(z_vx_d1)
  call devxlib_map(z_vy_d1)
#endif
  !
  ! init
  call random_number_cmplx( z_vx_h1 )
  call random_number_cmplx( z_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(z_vx_d1, z_vx_h1)
  call devxlib_memcpy_h2d(z_vy_d1, z_vy_h1)
  !
  ! host LinAlg
  z_scal_h1 = devxlib_xDOTU( ndim(1), z_vx_h1, 1, z_vy_h1, 1)
  z_scal_h3 = devxlib_xDOTC( ndim(1), z_vx_h1, 1, z_vy_h1, 1)
  !
  ! dev LinAlg
  z_scal_h2 = devxlib_xDOTU_gpu( ndim(1), z_vx_d1, 1, z_vy_d1, 1)
  z_scal_h4 = devxlib_xDOTC_gpu( ndim(1), z_vx_d1, 1, z_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  norm=10.0d0*abs(z_scal_h1)/sqrt(real(ndim(1)))
  !
  if ( abs(z_scal_h2-z_scal_h1)> thr_DP*norm .and. &
       abs(z_scal_h4-z_scal_h3)> thr_DP*norm ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( z_vx_h1 )
  deallocate( z_vy_h1 )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(z_vx_d1)
  call devxlib_unmap(z_vy_d1)
#endif
  deallocate( z_vx_d1 )
  deallocate( z_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  write(output_unit,"()")

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xAXPY (S) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( s_vx_h1(ndim(1)) )
  allocate( s_vy_h1(ndim(1)) )
  allocate( s_vy_h2(ndim(1)) )
  allocate( s_vx_d1(ndim(1)) )
  allocate( s_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(s_vx_d1)
  call devxlib_map(s_vy_d1)
#endif
  !
  ! init
  s_alpha=1.0
  call random_number( s_vx_h1 )
  call random_number( s_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(s_vx_d1, s_vx_h1)
  call devxlib_memcpy_h2d(s_vy_d1, s_vy_h1)
  !
  ! host LinAlg
  call devxlib_xAXPY( ndim(1), s_alpha, s_vx_h1, 1, s_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xAXPY_gpu( ndim(1), s_alpha, s_vx_d1, 1, s_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( s_vy_h2, s_vy_d1)
  !
  if ( any(abs(s_vy_h2-s_vy_h1)> thr_SP) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( s_vx_h1 )
  deallocate( s_vy_h1 )
  deallocate( s_vy_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(s_vx_d1)
  call devxlib_unmap(s_vy_d1)
#endif
  deallocate( s_vx_d1 )
  deallocate( s_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xAXPY (D) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( d_vx_h1(ndim(1)) )
  allocate( d_vy_h1(ndim(1)) )
  allocate( d_vy_h2(ndim(1)) )
  allocate( d_vx_d1(ndim(1)) )
  allocate( d_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(d_vx_d1)
  call devxlib_map(d_vy_d1)
#endif
  !
  ! init
  d_alpha=1.0
  call random_number( d_vx_h1 )
  call random_number( d_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(d_vx_d1, d_vx_h1)
  call devxlib_memcpy_h2d(d_vy_d1, d_vy_h1)
  !
  ! host LinAlg
  call devxlib_xAXPY( ndim(1), d_alpha, d_vx_h1, 1, d_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xAXPY_gpu( ndim(1), d_alpha, d_vx_d1, 1, d_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( d_vy_h2, d_vy_d1)
  !
  if ( any(abs(d_vy_h2-d_vy_h1)> thr_DP) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( d_vx_h1 )
  deallocate( d_vy_h1 )
  deallocate( d_vy_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(d_vx_d1)
  call devxlib_unmap(d_vy_d1)
#endif
  deallocate( d_vx_d1 )
  deallocate( d_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xAXPY (C) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( c_vx_h1(ndim(1)) )
  allocate( c_vy_h1(ndim(1)) )
  allocate( c_vy_h2(ndim(1)) )
  allocate( c_vx_d1(ndim(1)) )
  allocate( c_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(c_vx_d1)
  call devxlib_map(c_vy_d1)
#endif
  !
  ! init
  c_alpha=1.0
  call random_number_cmplx( c_vx_h1 )
  call random_number_cmplx( c_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(c_vx_d1, c_vx_h1)
  call devxlib_memcpy_h2d(c_vy_d1, c_vy_h1)
  !
  ! host LinAlg
  call devxlib_xAXPY( ndim(1), c_alpha, c_vx_h1, 1, c_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xAXPY_gpu( ndim(1), c_alpha, c_vx_d1, 1, c_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( c_vy_h2, c_vy_d1)
  !
  if ( any(abs(c_vy_h2-c_vy_h1)> thr_SP) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( c_vx_h1 )
  deallocate( c_vy_h1 )
  deallocate( c_vy_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(c_vx_d1)
  call devxlib_unmap(c_vy_d1)
#endif
  deallocate( c_vx_d1 )
  deallocate( c_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xAXPY (Z) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( z_vx_h1(ndim(1)) )
  allocate( z_vy_h1(ndim(1)) )
  allocate( z_vy_h2(ndim(1)) )
  allocate( z_vx_d1(ndim(1)) )
  allocate( z_vy_d1(ndim(1)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(z_vx_d1)
  call devxlib_map(z_vy_d1)
#endif
  !
  ! init
  z_alpha=1.0
  call random_number_cmplx( z_vx_h1 )
  call random_number_cmplx( z_vy_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(z_vx_d1, z_vx_h1)
  call devxlib_memcpy_h2d(z_vy_d1, z_vy_h1)
  !
  ! host LinAlg
  call devxlib_xAXPY( ndim(1), z_alpha, z_vx_h1, 1, z_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xAXPY_gpu( ndim(1), z_alpha, z_vx_d1, 1, z_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( z_vy_h2, z_vy_d1)
  !
  if ( any(abs(z_vy_h2-z_vy_h1)> thr_DP) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( z_vx_h1 )
  deallocate( z_vy_h1 )
  deallocate( z_vy_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(z_vx_d1)
  call devxlib_unmap(z_vy_d1)
#endif
  deallocate( z_vx_d1 )
  deallocate( z_vy_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0
  write(output_unit,"()")

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMV (S) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( s_vx_h1(ndim(2)) )
  allocate( s_vy_h1(ndim(1)) )
  allocate( s_mA_h1(ndim(1),ndim(2)) )
  allocate( s_vy_h2(ndim(1)) )
  allocate( s_vx_d1(ndim(2)) )
  allocate( s_vy_d1(ndim(1)) )
  allocate( s_mA_d1(ndim(1),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(s_vx_d1)
  call devxlib_map(s_vy_d1)
  call devxlib_map(s_mA_d1)
#endif
  !
  ! init
  s_alpha=1.0
  s_beta=1.0
  call random_number( s_vx_h1 )
  call random_number( s_vy_h1 )
  call random_number( s_mA_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(s_vx_d1, s_vx_h1)
  call devxlib_memcpy_h2d(s_vy_d1, s_vy_h1)
  call devxlib_memcpy_h2d(s_mA_d1, s_mA_h1)
  !
  ! host LinAlg
  call devxlib_xGEMV( 'N', ndim(1), ndim(2), s_alpha, s_mA_h1, ndim(1), s_vx_h1, 1, s_beta, s_vy_h1, 1)
  !
  ! dev LinAlg
  call start_prof("xGEMV_gpu")
  call devxlib_xGEMV_gpu( 'N', ndim(1), ndim(2), s_alpha, s_mA_d1, ndim(1), s_vx_d1, 1, s_beta, s_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  call stop_prof("xGEMV_gpu")
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( s_vy_h2, s_vy_d1)
  !
  call start_prof("check")
  norm=sqrt( sum(abs(s_vy_h1(:))**2)/real(size(s_vy_h1),kind=kind(s_vy_h1)) )
  if ( any(abs(s_vy_h2-s_vy_h1)> thr_SP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  call stop_prof("check")
  !
  deallocate( s_vx_h1 )
  deallocate( s_vy_h1 )
  deallocate( s_vy_h2 )
  deallocate( s_mA_h1 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(s_vx_d1)
  call devxlib_unmap(s_vy_d1)
  call devxlib_unmap(s_mA_d1)
#endif
  deallocate( s_vx_d1 )
  deallocate( s_vy_d1 )
  deallocate( s_mA_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMV (D) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( d_vx_h1(ndim(2)) )
  allocate( d_vy_h1(ndim(1)) )
  allocate( d_mA_h1(ndim(1),ndim(2)) )
  allocate( d_vy_h2(ndim(1)) )
  allocate( d_vx_d1(ndim(2)) )
  allocate( d_vy_d1(ndim(1)) )
  allocate( d_mA_d1(ndim(1),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(d_vx_d1)
  call devxlib_map(d_vy_d1)
  call devxlib_map(d_mA_d1)
#endif
  !
  ! init
  d_alpha=1.0
  d_beta=1.0
  call random_number( d_vx_h1 )
  call random_number( d_vy_h1 )
  call random_number( d_mA_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(d_vx_d1, d_vx_h1)
  call devxlib_memcpy_h2d(d_vy_d1, d_vy_h1)
  call devxlib_memcpy_h2d(d_mA_d1, d_mA_h1)
  !
  ! host LinAlg
  call devxlib_xGEMV( 'N', ndim(1), ndim(2), d_alpha, d_mA_h1, ndim(1), d_vx_h1, 1, d_beta, d_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xGEMV_gpu( 'N', ndim(1), ndim(2), d_alpha, d_mA_d1, ndim(1), d_vx_d1, 1, d_beta, d_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( d_vy_h2, d_vy_d1)
  !
  norm=sqrt( sum(abs(d_vy_h1(:))**2)/real(size(d_vy_h1),kind=kind(d_vy_h1)) )
  if ( any(abs(d_vy_h2-d_vy_h1)> thr_DP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( d_vx_h1 )
  deallocate( d_vy_h1 )
  deallocate( d_vy_h2 )
  deallocate( d_mA_h1 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(d_vx_d1)
  call devxlib_unmap(d_vy_d1)
  call devxlib_unmap(d_mA_d1)
#endif
  deallocate( d_vx_d1 )
  deallocate( d_vy_d1 )
  deallocate( d_mA_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMV (C) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( c_vx_h1(ndim(2)) )
  allocate( c_vy_h1(ndim(1)) )
  allocate( c_mA_h1(ndim(1),ndim(2)) )
  allocate( c_vy_h2(ndim(1)) )
  allocate( c_vx_d1(ndim(2)) )
  allocate( c_vy_d1(ndim(1)) )
  allocate( c_mA_d1(ndim(1),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(c_vx_d1)
  call devxlib_map(c_vy_d1)
  call devxlib_map(c_mA_d1)
#endif
  !
  ! init
  c_alpha=1.0
  c_beta=1.0
  call random_number_cmplx( c_vx_h1 )
  call random_number_cmplx( c_vy_h1 )
  call random_number_cmplx( c_mA_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(c_vx_d1, c_vx_h1)
  call devxlib_memcpy_h2d(c_vy_d1, c_vy_h1)
  call devxlib_memcpy_h2d(c_mA_d1, c_mA_h1)
  !
  ! host LinAlg
  call devxlib_xGEMV( 'N', ndim(1), ndim(2), c_alpha, c_mA_h1, ndim(1), c_vx_h1, 1, c_beta, c_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xGEMV_gpu( 'N', ndim(1), ndim(2), c_alpha, c_mA_d1, ndim(1), c_vx_d1, 1, c_beta, c_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( c_vy_h2, c_vy_d1)
  !
  norm=sqrt( sum(abs(c_vy_h1(:))**2)/real(size(c_vy_h1),kind=kind(c_vy_h1)) )
  if ( any(abs(c_vy_h2-c_vy_h1)> thr_SP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( c_vx_h1 )
  deallocate( c_vy_h1 )
  deallocate( c_vy_h2 )
  deallocate( c_mA_h1 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(c_vx_d1)
  call devxlib_unmap(c_vy_d1)
  call devxlib_unmap(c_mA_d1)
#endif
  deallocate( c_vx_d1 )
  deallocate( c_vy_d1 )
  deallocate( c_mA_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMV (Z) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( z_vx_h1(ndim(2)) )
  allocate( z_vy_h1(ndim(1)) )
  allocate( z_mA_h1(ndim(1),ndim(2)) )
  allocate( z_vy_h2(ndim(1)) )
  allocate( z_vx_d1(ndim(2)) )
  allocate( z_vy_d1(ndim(1)) )
  allocate( z_mA_d1(ndim(1),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(z_vx_d1)
  call devxlib_map(z_vy_d1)
  call devxlib_map(z_mA_d1)
#endif
  !
  ! init
  z_alpha=1.0
  z_beta=1.0
  call random_number_cmplx( z_vx_h1 )
  call random_number_cmplx( z_vy_h1 )
  call random_number_cmplx( z_mA_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(z_vx_d1, z_vx_h1)
  call devxlib_memcpy_h2d(z_vy_d1, z_vy_h1)
  call devxlib_memcpy_h2d(z_mA_d1, z_mA_h1)
  !
  ! host LinAlg
  call devxlib_xGEMV( 'N', ndim(1), ndim(2), z_alpha, z_mA_h1, ndim(1), z_vx_h1, 1, z_beta, z_vy_h1, 1)
  !
  ! dev LinAlg
  call devxlib_xGEMV_gpu( 'N', ndim(1), ndim(2), z_alpha, z_mA_d1, ndim(1), z_vx_d1, 1, z_beta, z_vy_d1, 1)
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h( z_vy_h2, z_vy_d1)
  !
  norm=sqrt( sum(abs(z_vy_h1(:))**2)/real(size(z_vy_h1),kind=kind(z_vy_h1)) )
  if ( any(abs(z_vy_h2-z_vy_h1)> thr_DP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( z_vx_h1 )
  deallocate( z_vy_h1 )
  deallocate( z_vy_h2 )
  deallocate( z_mA_h1 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(z_vx_d1)
  call devxlib_unmap(z_vy_d1)
  call devxlib_unmap(z_mA_d1)
#endif
  deallocate( z_vx_d1 )
  deallocate( z_vy_d1 )
  deallocate( z_mA_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0
  write(output_unit,"()")

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMM (S) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( s_mC_h1(ndim(1),ndim(2)) )
  allocate( s_mA_h1(ndim(1),ndim(3)) )
  allocate( s_mB_h1(ndim(3),ndim(2)) )
  allocate( s_mC_h2(ndim(1),ndim(2)) )
  !
  allocate( s_mC_d1(ndim(1),ndim(2)) )
  allocate( s_mA_d1(ndim(1),ndim(3)) )
  allocate( s_mB_d1(ndim(3),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(s_mC_d1)
  call devxlib_map(s_mA_d1)
  call devxlib_map(s_mB_d1)
#endif
  !
  ! init
  s_alpha=1.0
  s_beta=1.0
  call random_number( s_mA_h1 )
  call random_number( s_mB_h1 )
  call random_number( s_mC_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(s_mA_d1, s_mA_h1)
  call devxlib_memcpy_h2d(s_mB_d1, s_mB_h1)
  call devxlib_memcpy_h2d(s_mC_d1, s_mC_h1)
  !
  ! host LinAlg
  call start_prof("xGEMM_cpu")
  call devxlib_xGEMM( 'N', 'N', ndim(1), ndim(2), ndim(3), s_alpha, s_mA_h1, ndim(1), &
                            s_mB_h1, ndim(3), s_beta, s_mC_h1, ndim(1))
  call stop_prof("xGEMM_cpu")
  !
  ! dev LinAlg
  call start_prof("xGEMM_gpu")
  call devxlib_xGEMM_gpu( 'N', 'N', ndim(1), ndim(2), ndim(3), s_alpha, s_mA_d1, ndim(1), &
                                s_mB_d1, ndim(3), s_beta, s_mC_d1, ndim(1))
  call devxlib_async_synchronize(default_async_id)
  call stop_prof("xGEMM_gpu")
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h(s_mC_h2, s_mC_d1)
  !
  norm=sqrt( sum(abs(s_mC_h1(:,:))**2)/real(size(s_mC_h1),kind=kind(s_mC_h1)) )
  norm=norm*10.0
  !
  if ( any(abs(s_mC_h2-s_mC_h1)> thr_SP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( s_mA_h1 )
  deallocate( s_mB_h1 )
  deallocate( s_mC_h1 )
  deallocate( s_mC_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(s_mA_d1)
  call devxlib_unmap(s_mB_d1)
  call devxlib_unmap(s_mC_d1)
#endif
  !
  deallocate( s_mA_d1 )
  deallocate( s_mB_d1 )
  deallocate( s_mC_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMM (D) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( d_mC_h1(ndim(1),ndim(2)) )
  allocate( d_mA_h1(ndim(1),ndim(3)) )
  allocate( d_mB_h1(ndim(3),ndim(2)) )
  allocate( d_mC_h2(ndim(1),ndim(2)) )
  !
  allocate( d_mC_d1(ndim(1),ndim(2)) )
  allocate( d_mA_d1(ndim(1),ndim(3)) )
  allocate( d_mB_d1(ndim(3),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(d_mC_d1)
  call devxlib_map(d_mA_d1)
  call devxlib_map(d_mB_d1)
#endif
  !
  ! init
  d_alpha=1.0
  d_beta=1.0
  call random_number( d_mA_h1 )
  call random_number( d_mB_h1 )
  call random_number( d_mC_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(d_mA_d1, d_mA_h1)
  call devxlib_memcpy_h2d(d_mB_d1, d_mB_h1)
  call devxlib_memcpy_h2d(d_mC_d1, d_mC_h1)
  !
  ! host LinAlg
  call devxlib_xGEMM( 'N', 'N', ndim(1), ndim(2), ndim(3), d_alpha, d_mA_h1, ndim(1), &
                            d_mB_h1, ndim(3), d_beta, d_mC_h1, ndim(1))
  !
  ! dev LinAlg
  call devxlib_xGEMM_gpu( 'N', 'N', ndim(1), ndim(2), ndim(3), d_alpha, d_mA_d1, ndim(1), &
                                d_mB_d1, ndim(3), d_beta, d_mC_d1, ndim(1))
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h(d_mC_h2, d_mC_d1)
  !
  norm=sqrt( sum(abs(d_mC_h1(:,:))**2)/real(size(d_mC_h1),kind=kind(d_mC_h1)) )
  norm=norm*10.0
  !
  if ( any(abs(d_mC_h2-d_mC_h1)> thr_DP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( d_mA_h1 )
  deallocate( d_mB_h1 )
  deallocate( d_mC_h1 )
  deallocate( d_mC_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(d_mA_d1)
  call devxlib_unmap(d_mB_d1)
  call devxlib_unmap(d_mC_d1)
#endif
  !
  deallocate( d_mA_d1 )
  deallocate( d_mB_d1 )
  deallocate( d_mC_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMM (C) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( c_mC_h1(ndim(1),ndim(2)) )
  allocate( c_mA_h1(ndim(1),ndim(3)) )
  allocate( c_mB_h1(ndim(3),ndim(2)) )
  allocate( c_mC_h2(ndim(1),ndim(2)) )
  !
  allocate( c_mC_d1(ndim(1),ndim(2)) )
  allocate( c_mA_d1(ndim(1),ndim(3)) )
  allocate( c_mB_d1(ndim(3),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(c_mC_d1)
  call devxlib_map(c_mA_d1)
  call devxlib_map(c_mB_d1)
#endif
  !
  ! init
  c_alpha=1.0
  c_beta=1.0
  call random_number_cmplx( c_mA_h1 )
  call random_number_cmplx( c_mB_h1 )
  call random_number_cmplx( c_mC_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(c_mA_d1, c_mA_h1)
  call devxlib_memcpy_h2d(c_mB_d1, c_mB_h1)
  call devxlib_memcpy_h2d(c_mC_d1, c_mC_h1)
  !
  ! host LinAlg
  call devxlib_xGEMM( 'N', 'N', ndim(1), ndim(2), ndim(3), c_alpha, c_mA_h1, ndim(1), &
                            c_mB_h1, ndim(3), c_beta, c_mC_h1, ndim(1))
  !
  ! dev LinAlg
  call devxlib_xGEMM_gpu( 'N', 'N', ndim(1), ndim(2), ndim(3), c_alpha, c_mA_d1, ndim(1), &
                                c_mB_d1, ndim(3), c_beta, c_mC_d1, ndim(1))
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h(c_mC_h2, c_mC_d1)
  !
  norm=sqrt( sum(abs(c_mC_h1(:,:))**2)/real(size(c_mC_h1),kind=kind(c_mC_h1)) )
  norm=norm*10.0
  !
  if ( any(abs(c_mC_h2-c_mC_h1)> thr_SP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( c_mA_h1 )
  deallocate( c_mB_h1 )
  deallocate( c_mC_h1 )
  deallocate( c_mC_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(c_mA_d1)
  call devxlib_unmap(c_mB_d1)
  call devxlib_unmap(c_mC_d1)
#endif
  !
  deallocate( c_mA_d1 )
  deallocate( c_mB_d1 )
  deallocate( c_mC_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0

  !
  !=====================
  write(output_unit,"(/,3x,a)") "checking xGEMM (Z) ..."
  !=====================
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  !
  ! allocations
  allocate( z_mC_h1(ndim(1),ndim(2)) )
  allocate( z_mA_h1(ndim(1),ndim(3)) )
  allocate( z_mB_h1(ndim(3),ndim(2)) )
  allocate( z_mC_h2(ndim(1),ndim(2)) )
  !
  allocate( z_mC_d1(ndim(1),ndim(2)) )
  allocate( z_mA_d1(ndim(1),ndim(3)) )
  allocate( z_mB_d1(ndim(3),ndim(2)) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(z_mC_d1)
  call devxlib_map(z_mA_d1)
  call devxlib_map(z_mB_d1)
#endif
  !
  ! init
  z_alpha=1.0
  z_beta=1.0
  call random_number_cmplx( z_mA_h1 )
  call random_number_cmplx( z_mB_h1 )
  call random_number_cmplx( z_mC_h1 )
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(z_mA_d1, z_mA_h1)
  call devxlib_memcpy_h2d(z_mB_d1, z_mB_h1)
  call devxlib_memcpy_h2d(z_mC_d1, z_mC_h1)
  !
  ! host LinAlg
  call devxlib_xGEMM( 'N', 'N', ndim(1), ndim(2), ndim(3), z_alpha, z_mA_h1, ndim(1), &
                            z_mB_h1, ndim(3), z_beta, z_mC_h1, ndim(1))
  !
  ! dev LinAlg
  call devxlib_xGEMM_gpu( 'N', 'N', ndim(1), ndim(2), ndim(3), z_alpha, z_mA_d1, ndim(1), &
                                z_mB_d1, ndim(3), z_beta, z_mC_d1, ndim(1))
  call devxlib_async_synchronize(default_async_id)
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h(z_mC_h2, z_mC_d1)
  !
  norm=sqrt( sum(abs(z_mC_h1(:,:))**2)/real(size(z_mC_h1),kind=kind(z_mC_h1)) )
  norm=norm*10.0
  !
  if ( any(abs(z_mC_h2-z_mC_h1)> thr_DP*norm ) ) then
     !
     write(output_unit,"(3x,a)") "FAILED"
     nfail=nfail+1
  else
     write(output_unit,"(3x,a)") "passed"
     npass=npass+1
  endif
  !
  deallocate( z_mA_h1 )
  deallocate( z_mB_h1 )
  deallocate( z_mC_h1 )
  deallocate( z_mC_h2 )
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(z_mA_d1)
  call devxlib_unmap(z_mB_d1)
  call devxlib_unmap(z_mC_d1)
#endif
  !
  deallocate( z_mA_d1 )
  deallocate( z_mB_d1 )
  deallocate( z_mC_d1 )
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(output_unit,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0
  write(output_unit,"()")

  !
  ! summary
  !
  write(output_unit,"(/,a)") "# Test SUMMARY:"
  write(output_unit,"(3x,a,i5)") "# passed: ", npass
  write(output_unit,"(3x,a,i5)") "# failed: ", nfail
  write(output_unit,"()")

  write(output_unit,"(/,3x,a)") "Shutdown LinAlg ..."
  call dev_linalg_shutdown()
  write(output_unit,"()")

  contains

     subroutine random_number_cmplx_c1(array)
        implicit none
        complex(real32) :: array(:)
        real(real32), allocatable :: rtmp(:)
        !
        allocate(rtmp,mold=real(array))
        call random_number(rtmp)
        array=rtmp
        call random_number(rtmp)
        array=array+ci_sp*rtmp
        deallocate(rtmp)
     end subroutine

     subroutine random_number_cmplx_c2(array)
        implicit none
        complex(real32) :: array(:,:)
        real(real32), allocatable :: rtmp(:,:)
        !
        allocate(rtmp,mold=real(array))
        call random_number(rtmp)
        array=rtmp
        call random_number(rtmp)
        array=array+ci_sp*rtmp
        deallocate(rtmp)
     end subroutine

     subroutine random_number_cmplx_z1(array)
        implicit none
        complex(real64) :: array(:)
        real(real64), allocatable :: rtmp(:)
        !
        allocate(rtmp,mold=real(array))
        call random_number(rtmp)
        array=rtmp
        call random_number(rtmp)
        array=array+ci_sp*rtmp
        deallocate(rtmp)
     end subroutine

     subroutine random_number_cmplx_z2(array)
        implicit none
        complex(real64) :: array(:,:)
        real(real64), allocatable :: rtmp(:,:)
        !
        allocate(rtmp,mold=real(array))
        call random_number(rtmp)
        array=rtmp
        call random_number(rtmp)
        array=array+ci_sp*rtmp
        deallocate(rtmp)
     end subroutine
end program test_linalg
