安装并测试cuda版本的PETSc

• cuda如果是通过apt安装的，与cuda相关cuda路径如下：
  1. /usr/bin
  2. /usr/lib/cuda
  3. /usr/nvidia-cuda-toolkit
• cuda如果是通过*.run文件安装的，默认情况下相关路径为：
  1. /usr/local/cuda
  2. /usr/local/cuda-10.2

如有错误欢迎指正。我是通过apt安装的。

安装openmpi

export SOFTWARE=$HOME/software

cd $SOFTWARE
tar -jxvf openmpi*.bz2
rm openmpi*.bz2
mv openmpi* openmpi-source
cd $SOFTWARE/openmpi-source

./configure --prefix=$SOFTWARE/openmpi --with-cuda=/usr

make -j64 install

添加路径

export SOFTWARE=$HOME/software
export PATH=$SOFTWARE/openmpi/bin:$PATH
export LD_LIBRARY_PATH=$SOFTWARE/openmpi/lib:$LD_LIBRARY_PATH
export C_INCLUDE_PATH=$SOFTWARE/openmpi/include:$C_INCLUDE_PATH

安装petsc

unzip petsc*.zip
rm petsc*.zip
mv petsc* petsc
cd $SOFTWARE/petsc

export PETSC_DIR=$SOFTWARE/petsc
export PETSC_ARCH=linux-opt

python3 configure \
--CC=$SOFTWARE/openmpi/bin/mpicc \
--CXX=$SOFTWARE/openmpi/bin/mpicxx \
--FC=$SOFTWARE/openmpi/bin/mpif90 \
--COPTFLAGS="-O3" \
--CXXOPTFLAGS="-O3" \
--FOPTFLAGS="-O3" \
--CUDAOPTFLAGS="-O3" \
--PETSC_ARCH=$PETSC_ARCH \
--with-debugging=0 \
--with-cuda \
--with-cudac=nvcc \
--with-cuda-dir=/usr/lib/cuda

make -j64 all
make check

添加路径

# necessary variables
export PETSC_DIR=$SOFTWARE/petsc
export PETSC_ARCH=linux-opt

# library path
export PATH=/public/home/npuheart/ibamr/petsc/linux-opt/bin/:$PATH
export C_INCLUDE_PATH=$SOFTWARE/petsc/linux-opt/include:$C_INCLUDE_PATH
export LD_LIBRARY_PATH=$SOFTWARE/petsc/linux-opt/lib:$LD_LIBRARY_PATH

# these might be unnecessary
export C_INCLUDE_PATH=$SOFTWARE/petsc/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$SOFTWARE/petsc/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$SOFTWARE/petsc/linux-opt/include:$CPLUS_INCLUDE_PATH

测试

# 这是一个简单的例子
cd $SOFTWARE/petsc/src/ksp/ksp/tutorials


# 编译
make ex12


# cpu测试
time mpirun -np 1 ex12 -n 200 -m 200 -pc_type none
Norm of error 0.574264 iterations 1630

real    0m5.824s
user    0m2.454s
sys     0m2.697s
# cpu测试
time mpirun -np 8 ex12 -n 200 -m 200 -pc_type none
Norm of error 0.574264 iterations 1630

real    0m5.568s
user    0m8.582s
sys     0m4.897s


# gpu测试
time mpirun -np 1 ex12 -n 200 -m 200 -pc_type none -vec_type cuda -mat_type aijcusparse
Norm of error 0.574264 iterations 1630

real    0m4.934s
user    0m1.181s
sys     0m0.871s
# gpu测试
time mpirun -np 8 ex12 -n 200 -m 200 -pc_type none -vec_type cuda -mat_type aijcusparse
Norm of error 0.574264 iterations 1630

real    0m17.775s
user    1m4.660s
sys     0m12.970s

the alternation of gcc-9 and gcc-8 ( including g++, gfortran )

sudo update-alternatives --install /usr/bin/gcc gcc /ur/bin/gcc-8 100
sudo update-alternatives --install /usr/bin/g++ g++ /ur/bin/g++-8 100
sudo update-alternatives --install /usr/bin/gfortran gfortran /ur/bin/gcc-8 100