lammps编译（2Aug2023、intel2020、rtx4070ti）

说明：

[root@node101 ~]# cat /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)
[root@node101 ~]# gcc -v
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/4.8.5/lto-wrapper
Target: x86_64-redhat-linux
Configured with: ../configure --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-bootstrap --enable-shared --enable-threads=posix --enable-checking=release --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-linker-hash-style=gnu --enable-languages=c,c++,objc,obj-c++,java,fortran,ada,go,lto --enable-plugin --enable-initfini-array --disable-libgcj --with-isl=/builddir/build/BUILD/gcc-4.8.5-20150702/obj-x86_64-redhat-linux/isl-install --with-cloog=/builddir/build/BUILD/gcc-4.8.5-20150702/obj-x86_64-redhat-linux/cloog-install --enable-gnu-indirect-function --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
Thread model: posix
gcc version 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC)
[root@node101 ~]# which mpirun
/opt/gpuApp/ompi/bin/mpirun
[root@node101 ~]# which icc
/opt/intel/compilers_and_libraries_2020.1.211/linux/bin/intel64/icc
[root@node101 ~]# which nvcc
/usr/local/cuda-12.3/bin/nvcc
[root@node101 ~]# lscpu
Architecture:          x86_64
CPU op-mode(s):        32-bit, 64-bit
Byte Order:            Little Endian
CPU(s):                32
On-line CPU(s) list:   0-31
Thread(s) per core:    2
Core(s) per socket:    16
Socket(s):             1
NUMA node(s):          1
Vendor ID:             AuthenticAMD
CPU family:            23
Model:                 49
Model name:            AMD EPYC 7302 16-Core Processor
Stepping:              0
CPU MHz:               1500.000
CPU max MHz:           3000.0000
CPU min MHz:           1500.0000
BogoMIPS:              6000.34
Virtualization:        AMD-V
L1d cache:             32K
L1i cache:             32K
L2 cache:              512K
L3 cache:              16384K
NUMA node0 CPU(s):     0-31
Flags:                 fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc art rep_good nopl nonstop_tsc extd_apicid aperfmperf eagerfpu pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_l2 cpb cat_l3 cdp_l3 hw_pstate sme retpoline_amd ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif umip overflow_recov succor smca
[root@node101 ~]# free -g
              total        used        free      shared  buff/cache   available
Mem:            251           5         227           0          18         244
Swap:           127           0         127
[root@node101 ~]#
 

lammps支持单精度，也支持双精度。受限于4070Ti，其双精度能力很差，故本次使用单精度方式进行使用。

显卡的SM值可以通过cuda自带的工具查询：

[root@node101 tools]#ls /usr/local/cuda/samples/1_Utilities/deviceQuery

deviceQuery deviceQuery.cpp deviceQuery.o Makefile NsightEclipse.xml readme.txt

[root@node101 tools]#cd /usr/local/cuda/samples/1_Utilities/deviceQuery

[root@node101 deviceQuery]#./deviceQuery

1、环境文件

cat << EOF > ~/lammps-gpu-env.sh

#!/bin/bash

source /opt/intel/compilers_and_libraries_2020/linux/bin/compilervars.sh intel64

export PATH=/usr/local/cuda-12.3/bin:$PATH

export LD_LIBRARY_PATH=/usr/local/cuda-12.3/targets/x86_64-linux/lib:$LD_LIBRARY_PATH

export C_INCLUDE_PATH=/usr/local/cuda-12.3/targets/x86_64-linux/include:$C_INCLUDE_PATH

EOF

2、gdrcopy

cd gpu-lammps/

tar -zxvf gdrcopy-2.0.tar.gz

cd gdrcopy-2.0/

mkdir -p /opt/gpuApp/gdrcopy/include

mkdir -p /opt/gpuApp/gdrcopy/lib64

make PREFIX=/opt/gpuApp/gdrcopy lib lib_install

cat << EOF >> ~/lammps-gpu-env.sh

export PATH=/opt/gpuApp/gdrcopy/include:\$PATH

export CPATH=/opt/gpuApp/gdrcopy/include:\$CPATH

export LD_LIBRARY_PATH=/opt/gpuApp/gdrcopy/lib64:\$LD_LIBRARY_PATH

EOF

3、ucx

cd ~/gpu-lammps/

tar -zxvf ucx-1.7.0.tar.gz

cd ucx-1.7.0/

./configure --prefix=/opt/gpuApp/ucx --enable-optimizations --disable-logging --disable-debug --disable-assertions --disable-params-check --disable-doxygen-doc --with-cuda=/usr/local/cuda --with-gdrcopy=/opt/gpuApp/gdrcopy/ --with-verbs --with-rdmacm

……………………………….

configure: =========================================================

configure: UCX build configuration:

configure: Preprocessor flags:   -DCPU_FLAGS="|avx" -I${abs_top_srcdir}/src -I${abs_top_builddir} -I${abs_top_builddir}/src

configure:            C flags:   -O3 -g -Wall -Werror -mavx

configure:          C++ flags:   -O3 -g -Wall -Werror -mavx

configure:       Multi-thread:   Disabled

configure:          MPI tests:   Disabled

configure:      Devel headers:

configure:        UCT modules:   < cuda ib rdmacm cma >

configure:       CUDA modules:   < gdrcopy >

configure:       ROCM modules:   < >

configure:         IB modules:   < >

configure:        UCM modules:   < cuda >

configure:       Perf modules:   < cuda >

configure: =========================================================

…………..

cat << EOF >> ~/lammps-gpu-env.sh

export PATH=/opt/gpuApp/ucx/bin:\$PATH

export LD_LIBRARY_PATH=/opt/gpuApp/ucx/lib:\$LD_LIBRARY_PATH

EOF

4、openmpi

[root@node101 gpu-lammps]# cd ~/gpu-lammps/

[root@node101 gpu-lammps]# tar -xvf openmpi-4.1.6.tar

[root@node101 gpu-lammps]# cd openmpi-4.1.6/

[root@node101 openmpi-4.1.6]# ./configure --prefix=/opt/gpuApp/ompi --enable-mpirun-prefix-by-default --enable-cuda --enable-dlopen --enable-weak-symbols --enable-heterogeneous --enable-binaries --enable-script-wrapper-compilers --enable-orterun-prefix-by-default --enable-mca-no-build=btl-uct --with-cuda --with-pmix --with-verbs --with-ucx=/opt/gpuApp/ucx

…………

Open MPI configuration:

-----------------------

Version: 4.1.6

Build MPI C bindings: yes

Build MPI C++ bindings (deprecated): no

Build MPI Fortran bindings: mpif.h, use mpi

MPI Build Java bindings (experimental): no

Build Open SHMEM support: yes

Debug build: no

Platform file: (none)

Miscellaneous

-----------------------

CUDA support: yes

HWLOC support: internal

Libevent support: internal

Open UCC: no

PMIx support: Internal

Transports

-----------------------

Cisco usNIC: no

Cray uGNI (Gemini/Aries): no

Intel Omnipath (PSM2): no

Intel TrueScale (PSM): no

Mellanox MXM: no

Open UCX: yes

OpenFabrics OFI Libfabric: no

OpenFabrics Verbs: yes

Portals4: no

Shared memory/copy in+copy out: yes

Shared memory/Linux CMA: yes

Shared memory/Linux KNEM: no

Shared memory/XPMEM: no

TCP: yes

Resource Managers

-----------------------

Cray Alps: no

Grid Engine: no

LSF: no

Moab: no

Slurm: yes

ssh/rsh: yes

Torque: no

OMPIO File Systems

-----------------------

DDN Infinite Memory Engine: no

Generic Unix FS: yes

IBM Spectrum Scale/GPFS: no

Lustre: no

PVFS2/OrangeFS: no

[root@node101 openmpi-4.1.6]# make -j 32

[root@node101 openmpi-4.1.6]# make install

[root@node101 openmpi-4.1.6]# cat << EOF >> ~/lammps-gpu-env.sh

export PATH=/opt/gpuApp/ompi/bin:\$PATH

export LD_LIBRARY_PATH=/opt/gpuApp/ompi/lib:\$LD_LIBRARY_PATH

export INCLUDE=/opt/gpuApp/ompi/include:\$INCLUDE

EOF

[root@node101 openmpi-4.1.6]#

5、lammps-cpu

[root@node101 gpu-lammps]# tar -zxvf lammps-2Aug2023.tar.gz

[root@node101 gpu-lammps]# cd lammps-2Aug2023/src

[root@node101 src]#source ~/lammps-gpu-env.sh

[root@node101 src]# make yes-all

[root@node101 src]# make no-lib

[root@node101 src]# cp MAKE/OPTIONS/Makefile.intel_cpu_openmpi MAKE/Makefile.intel

[root@node101 src]# make -j 32 intel

[root@node101 src]# cp lmp_intel lmp_intel_cpu

6、lammps-gpu

[root@node101 gpu-lammps]# cd lammps-2Aug2023/lib/gpu/

[root@node101 gpu]#source ~/lammps-gpu-env.sh

[root@node101 gpu]# vi Makefile.linux                         ##修改SM和CUDA_PRECISION[强撞1] 

[root@node101 gpu]# make -f Makefile.linux               ##编译GPU库

[root@node101 gpu]# ./nvc_get_devices

Found 1 platform(s).

CUDA Driver Version:                           12.30

Device 0: "NVIDIA GeForce RTX 4070 Ti"

  Type of device:                                GPU

  Compute capability:                            8.9

  Double precision support:                      Yes

  Total amount of global memory:                 11.7281 GB

  Number of compute units/multiprocessors:       60

  Number of cores:                               11520

  Total amount of constant memory:               65536 bytes

  Total amount of local/shared memory per block: 49152 bytes

  Total number of registers available per block: 65536

  Warp size:                                     32

  Maximum number of threads per block:           1024

  Maximum group size (# of threads per block)    1024 x 1024 x 64

  Maximum item sizes (# threads for each dim)    2147483647 x 65535 x 65535

  Maximum memory pitch:                          2147483647 bytes

  Texture alignment:                             512 bytes

  Clock rate:                                    2.61 GHz

  Run time limit on kernels:                     No

  Integrated:                                    No

  Support host page-locked memory mapping:       Yes

  Compute mode:                                  Default

  Concurrent kernel execution:                   Yes

  Device has ECC support enabled:                No

Device 1: "NVIDIA GeForce RTX 4070 Ti"

  Type of device:                                GPU

  Compute capability:                            8.9

  Double precision support:                      Yes

  Total amount of global memory:                 11.7281 GB

  Number of compute units/multiprocessors:       60

  Number of cores:                               11520

  Total amount of constant memory:               65536 bytes

  Total amount of local/shared memory per block: 49152 bytes

  Total number of registers available per block: 65536

  Warp size:                                     32

  Maximum number of threads per block:           1024

  Maximum group size (# of threads per block)    1024 x 1024 x 64

  Maximum item sizes (# threads for each dim)    2147483647 x 65535 x 65535

  Maximum memory pitch:                          2147483647 bytes

  Texture alignment:                             512 bytes

  Clock rate:                                    2.61 GHz

  Run time limit on kernels:                     No

  Integrated:                                    No

  Support host page-locked memory mapping:       Yes

  Compute mode:                                  Default

  Concurrent kernel execution:                   Yes

  Device has ECC support enabled:                No

Device 2: "NVIDIA GeForce RTX 4070 Ti"

  Type of device:                                GPU

  Compute capability:                            8.9

  Double precision support:                      Yes

  Total amount of global memory:                 11.7281 GB

  Number of compute units/multiprocessors:       60

  Number of cores:                               11520

  Total amount of constant memory:               65536 bytes

  Total amount of local/shared memory per block: 49152 bytes

  Total number of registers available per block: 65536

  Warp size:                                     32

  Maximum number of threads per block:           1024

  Maximum group size (# of threads per block)    1024 x 1024 x 64

  Maximum item sizes (# threads for each dim)    2147483647 x 65535 x 65535

  Maximum memory pitch:                          2147483647 bytes

  Texture alignment:                             512 bytes

  Clock rate:                                    2.61 GHz

  Run time limit on kernels:                     No

  Integrated:                                    No

  Support host page-locked memory mapping:       Yes

  Compute mode:                                  Default

  Concurrent kernel execution:                   Yes

  Device has ECC support enabled:                No

Device 3: "NVIDIA GeForce RTX 4070 Ti"

  Type of device:                                GPU

  Compute capability:                            8.9

  Double precision support:                      Yes

  Total amount of global memory:                 11.7281 GB

  Number of compute units/multiprocessors:       60

  Number of cores:                               11520

  Total amount of constant memory:               65536 bytes

  Total amount of local/shared memory per block: 49152 bytes

  Total number of registers available per block: 65536

  Warp size:                                     32

  Maximum number of threads per block:           1024

  Maximum group size (# of threads per block)    1024 x 1024 x 64

  Maximum item sizes (# threads for each dim)    2147483647 x 65535 x 65535

  Maximum memory pitch:                          2147483647 bytes

  Texture alignment:                             512 bytes

  Clock rate:                                    2.61 GHz

  Run time limit on kernels:                     No

  Integrated:                                    No

  Support host page-locked memory mapping:       Yes

  Compute mode:                                  Default

  Concurrent kernel execution:                   Yes

  Device has ECC support enabled:                No

[root@node101 gpu]# cd ../../src

[root@node101 src]#make package-status

[root@node101 src]#make yes-gpu

[root@node101 src]#make no-amoeba

[root@node101 src]#make clean-all

[root@node101 src]#make clean-machine

[root@node101 src]#make clean-intel

[root@node101 src]#make -j 32 intel

[root@node101 src]#cp lmp_intel lmp_intel_gpu

7、测试

7.1cpu

source /opt/gpuApp/lammps-gpu-env.sh

mpirun -np 12 /opt/gpuApp/lammps/lmp_intel_cpu -in in.NHO

7.2 4core_1gpu

source /opt/gpuApp/lammps-gpu-env.sh

mpirun -np 4 /opt/gpuApp/lammps/lmp_intel_cuda -sf gpu -pk gpu 1 -in in.NHO

GPU状态：

7.3 16core_1gpu

source /opt/gpuApp/lammps-gpu-env.sh

mpirun -np 16 /opt/gpuApp/lammps/lmp_intel_cuda -sf gpu -pk gpu 1 -in in.NHO

GPU状态：

7.4 16core_4gpu

source /opt/gpuApp/lammps-gpu-env.sh

mpirun -np 16 /opt/gpuApp/lammps/lmp_intel_cuda -sf gpu -pk gpu 4 -in in.NHO

GPU状态：