Works with nvidia on raven now

2025-ACH-STRUPHY/00_GPU_TESTS/salloc_raven_apudev.sh

+salloc --gres=gpu:a100:1 --cpus-per-task=1 --mem=20000 --time=00:15:00 --partition=gpudev --job-name=test_gpu_raven
+
+module purge
+module load gcc/14 intel/21.2.0 impi/2021.2 likwid/5.1
+module load nvhpcsdk/25 cuda/12.6-nvhpcsdk_25
+source ~/virtual_envs/env_pyccel/bin/activate
+
+LIKWID_PREFIX=$(realpath $(dirname $(which likwid-topology))/..)                 # Set LIKWID prefix
+export LD_LIBRARY_PATH=$LIKWID_PREFIX/lib                                        # Update LD_LIBRARY_PATH for LIKWID
+
+# Set OpenMP variables
+export OFFLOAD_TRACK_ALLOCATION_TRACES=true
+export HSA_XNACK=1 
+export LIBOMPTARGET_KERNEL_TRACE=1
+export LIBOMPTARGET_DEBUG=1
+# export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
+export OMP_TARGET_OFFLOAD=MANDATORY
+echo $OMP_TARGET_OFFLOAD
\ No newline at end of file

2025-ACH-STRUPHY/00_GPU_TESTS/slurm_raven_gpu_check.sh

 #!/bin/bash -l
 # Standard output and error:
-#SBATCH -o ./job.out.%j
-#SBATCH -e ./job.err.%j
+#SBATCH -o ./jobs/job.out.%j
+#SBATCH -e ./jobs/job.err.%j
 # Initial working directory:
 #SBATCH -D ./
 # Job name
-#SBATCH -J test_gpu
-# # SBATCH --partition=gpudev
+#SBATCH -J test_gpu_raven
+#SBATCH --partition=gpudev
 #SBATCH --ntasks=1
-#SBATCH --constraint="apu"
-#
 # --- default case: use a single GPU on a shared node ---
-#SBATCH --gres=gpu:1
+#SBATCH --gres=gpu:a100:1
 #SBATCH --cpus-per-task=1
-#SBATCH --mem=40000
+#SBATCH --mem=20000
 #
+#SBATCH --time=00:10:00
+
 #SBATCH --mail-type=none
-#SBATCH --mail-user=userid@example.mpg.de
-#SBATCH --time=00:05:00
+#SBATCH --mail-user=max.lindqvist@ipp.mpg.de
+
 
 echo "=========================================================================="
 echo "Script Path: $(realpath "$0")"
@@ -29,11 +29,12 @@ echo "==========================================================================
 FOLDER_NAME=jobs/job_${SLURM_JOB_NAME}_${SLURM_JOBID}
 mkdir -p $FOLDER_NAME
 cp $0 $FOLDER_NAME
-cp -r tests/* $FOLDER_NAME
+cp -r tests/ $FOLDER_NAME
 cd $FOLDER_NAME
 
 module purge
-module load gcc/14 cuda/12.2 intel/21.2.0 impi/2021.2 likwid/5.1
+module load gcc/14 intel/21.2.0 impi/2021.2 likwid/5.1
+module load nvhpcsdk/25 cuda/12.6-nvhpcsdk_25
 
 source ~/virtual_envs/env_pyccel/bin/activate
 pip install psutil
@@ -52,24 +53,30 @@ export OMP_TARGET_OFFLOAD=MANDATORY
 echo $OMP_TARGET_OFFLOAD
 
 # Print hardware information
-sh hardware_check/print_hardware_information.sh > hardware_check/hardware_info.txt
-python hardware_check/print_hardware_information.py >> hardware_check/hardware_info.txt
+sh tests/hardware_check/print_hardware_information.sh > hardware_check/hardware_info.txt
+python tests/hardware_check/print_hardware_information.py >> hardware_check/hardware_info.txt
 
 echo "=== Compiling hello_openmp_gpu ==="
 # Compile
-gcc -fopenmp -o C/hello_openmp_gpu C/hello_openmp_gpu.c
+nvc -mp=gpu -o tests/C/hello_openmp_gpu tests/C/hello_openmp_gpu.c
 
 echo "=== Running ./hello_openmp_gpu ==="
-./C/hello_openmp_gpu > C/output_hello_openmp_gpu.txt
+./tests/C/hello_openmp_gpu > tests/C/hello_openmp_gpu.out
+
+# ------------ CUDA ------------ #
 
 echo "=== Compiling hello_cuda_gpu ==="
-nvcc -o C/hello_cuda_gpu C/hello_cuda_gpu.c
+nvc -o tests/C/hello_cuda_gpu tests/C/hello_cuda_gpu.c
 
 echo "=== Running ./hello_cuda_gpu ==="
-./C/hello_cuda_gpu > C/output_hello_cuda_gpu.txt
+./tests/C/hello_cuda_gpu > tests/C/hello_cuda_gpu.out
 
 # ------------ Python ------------ #
-cd python
+cd tests/python
+
+# Use nvfortran instead of nvfort
+ln -s $(which nvfortran) ~/bin/nvfort
+export PATH=~/bin:$PATH
 
 # Pyccel hello world
 pyccel --language c hello_world.py
@@ -78,7 +85,7 @@ pyccel --language c hello_world.py
 echo "=== Compiling pyccel_kernels ==="
 make clean
 make LANGUAGE=fortran COMPILER=compiler_setups/compiler_nvidia.json PYTHON_FILE=pyccel_kernels.py
-python test_pyccel_kernels.py
+nsys profile --stats=true python test_pyccel_kernels.py > test_pyccel_kernels.out
 
 # mv ../job.out.${SLURM_JOBID} .
 # mv ../job.err.${SLURM_JOBID} .

2025-ACH-STRUPHY/00_GPU_TESTS/tests/python/compiler_setups/compiler_nvidia.json

@@ -18,7 +18,7 @@
         ],
         "openmp": {
             "flags": [
-                "-mp"
+                "-mp=gpu"
             ]
         },
         "openacc": {
@@ -78,7 +78,7 @@
         ],
         "openmp": {
             "flags": [
-                "-mp"
+                "-mp=gpu"
             ]
         },
         "openacc": {

2025-ACH-STRUPHY/00_GPU_TESTS/tests/python/test_pyccel_kernels.py

@@ -3,7 +3,7 @@ import numpy as np
 from pyccel_kernels import matmul_cpu, matmul_gpu
 
 def test_matmul():
-    N: int = 2**10
+    N: int = 2**12
     A = np.zeros((N, N))
     B = np.zeros((N, N))
     for i in range(N):
@@ -21,6 +21,8 @@ def test_matmul():
     start_cpu = time.time()
     matmul_cpu(A, B, C_cpu)
     elapsed_cpu = time.time() - start_cpu
+    print("warming up gpu")
+    matmul_gpu(A, B, C_gpu)
 
     print('Start matmul_gpu')
     # Time GPU matrix multiplication.