使用 AWS ParallelCluster 运行 Ansys Fluent 的最佳实践

yum -y update

yum install -y dkms zlib-devel libXext-devel libGLU-devel libXt-devel libXrender-devel libXinerama-devel libpng-devel libXrandr-devel libXi-devel libXft-devel libjpeg-turbo-devel libXcursor-devel readline-devel ncurses-devel python python-devel cmake qt-devel qt-assistant mpfr-devel gmp-devel htop wget screen vim xorg-x11-drv-dummy xorg-x11-server-utils libXp.x86_64 xorg-x11-fonts-cyrillic.noarch xterm.x86_64 openmotif.x86_64 compat-libstdc++-33.x86_64 libstdc++.x86_64 libstdc++.i686 gcc-c++.x86_64 compat-libstdc++-33.i686 libstdc++-devel.x86_64 libstdc++-devel.i686 compat-gcc-34.x86_64 gtk2.i686 libXxf86vm.i686 libSM.i686 libXt.i686 xorg-x11-fonts-ISO8859-1-75dpi.no xorg-x11-fonts-iso8859-1-75dpi.no libXext gdm gnome-session gnome-classic-session gnome-session-xsession xorg-x11-server-Xorg xorg-x11-drv-dummy xorg-x11-fonts-Type1 xorg-x11-utils gnome-terminal gnu-free-fonts-common gnu-free-mono-fonts gnu-free-sans-fonts gnu-free-serif-fonts alsa-plugins-pulseaudio alsa-utils

yum -y groupinstall "GNOME Desktop"

yum -y erase initial-setup gnome-initial-setup initial-setup-gui

#a reboot here may be helpful in case the kernel has been updated

#this will disable the ssh host key checking
#usually this may not be needed, but with some specific configuration Fluent may require this setting.
cat <<\EOF >> /etc/ssh/ssh_config
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
EOF


#set higher limits, usefull when running Fluent (and in general HPC applications) on multiple nodes via mpi
cat <<\EOF >> /etc/security/limits.conf
* hard memlock unlimited
* soft memlock unlimited
* hard stack 1024000
* soft stack 1024000
* hard nofile 1024000
* soft nofile 1024000
EOF

#stop and disable the firewall
systemctl disable firewalld
systemctl stop firewalld

#install the latest ENA driver, ATM 2.1.1
cd /tmp
wget https://github.com/amzn/amzn-drivers/archive/ena_linux_2.1.1.tar.gz
tar zxvf ena_linux_2.1.1.tar.gz
mv amzn-drivers-ena_linux_2.1.1 /usr/src/ena-2.1.1

cat <<EOF > /usr/src/ena-2.1.1/dkms.conf
PACKAGE_NAME="ena"
PACKAGE_VERSION="2.1.1"
AUTOINSTALL="yes"
REMAKE_INITRD="yes"
BUILT_MODULE_LOCATION[0]="kernel/linux/ena"
BUILT_MODULE_NAME[0]="ena"
DEST_MODULE_LOCATION[0]="/updates"
DEST_MODULE_NAME[0]="ena"
CLEAN="cd kernel/linux/ena; make clean"
MAKE="cd kernel/linux/ena; make BUILD_KERNEL=\${kernelver}"
EOF

dkms add -m ena -v 2.1.1
dkms build -m ena -v 2.1.1
dkms install -m ena -v 2.1.1

dracut -f —add-drivers ena
#reboot again, and make sure that after the reboot the ena driver is up to date (run modinfo ena to check)


#install the latest version of NICE DCV (at the moment it is 2017.4)
cd /tmp
wget https://d1uj6qtbmh3dt5.cloudfront.net/server/nice-dcv-2017.4-6898-el7.tgz
tar xzvf nice-dcv-2017.4-6898-el7.tgz
cd nice-dcv-2017.4-6898-el7
yum install -y nice-dcv-server-2017.4.6898-1.el7.x86_64.rpm nice-dcv-gltest-2017.4.216-1.el7.x86_64.rpm nice-xdcv-2017.4.210-1.el7.x86_64.rpm 

#install this additional package only in case you are running on an instance equipped with GPU
yum install -y nice-dcv-gl-2017.4.490-1.el7.i686.rpm nice-dcv-gl-2017.4.490-1.el7.x86_64.rpm

# Add the line "blacklist = /usr/bin/Xorg" to section [gl] of /etc/dcv/dcv-gl.conf
# to fix an incompatibility introduced with the latest versions of Xorg and Nvidia driver
sed -i 's|\[gl\]|&\nblacklist = /usr/bin/Xorg|' /etc/dcv/dcv-gl.conf

#Clean up the instance before creating the AMI.
/usr/local/sbin/ami_cleanup.sh

#shutdown the instance
shutdown -h now

aws ec2 create-image --instance-id i-1234567890abcdef0 --name "Fluent-AMI-v1" --description "This is my first Ansys Fluent AMI"

{
"ImageId": "ami-1a2b3c4d5e6f7g"
}

[aws]
aws_region_name = <your-preferred-region>

[global]
sanity_check = true
cluster_template = fluent_cluster_test1
update_check = true


[vpc vpc-us-east-1]
vpc_id = vpc-<VPC-ID>
master_subnet_id = subnet-<Subnet-ID>
additional_sg=sg-<Security-Group-ID>


[cluster fluent_cluster]
key_name = <Key-Name>
vpc_settings = vpc-us-east-1
compute_instance_type=c5n.18xlarge
master_instance_type=g3.4xlarge
initial_queue_size = 0
max_queue_size = 10
maintain_initial_size = true
scheduler=sge
cluster_type = ondemand
s3_read_write_resource=arn:aws:s3:::<Your-S3-Bucket>*
post_install = s3://<Your-S3-Bucket>/fluent-post-install.sh
placement_group = DYNAMIC
placement = compute
master_root_volume_size = 64
compute_root_volume_size = 20
base_os = centos7
extra_json = { "cluster" : {"cfn_scheduler_slots" : "cores" } }
tags = {"Name" : "fluent_cluster_test1"}
fsx_settings = parallel-fs
custom_ami = ami-<AMI-ID>


[fsx parallel-fs]
shared_dir = /fsx
storage_capacity = 3600
import_path = s3://<Your-S3-Bucket>
imported_file_chunk_size = 1024
export_path = s3://<Your-S3-Bucket>/export

#!/bin/bash

#source the AWS ParallelCluster profile
. /etc/parallelcluster/cfnconfig

#disable hyper-threading
for cpunum in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | cut -s -d, -f2- | tr ',' '\n' | sort -un); do
echo 0 > /sys/devices/system/cpu/cpu$cpunum/online
done


case "${cfn_node_type}" in
MasterServer)

#save the instance type
instanceType=$(curl http://169.254.169.254/latest/meta-data/instance-type)
if [[ $instanceType == *"g3"* ]]; then

  # configure Xorg to use the Nvidia GPU with the right driver
  nvidia-xconfig -preserve-busid -enable-all-gpus

  #Configure the GPU settings to be persistent
  nvidia-persistenced

  #Disable the autoboost feature for all GPUs on the instance
  nvidia-smi -auto-boost-default=0

  #Set all GPU clock speeds to their maximum frequency.
  nvidia-smi -ac 2505,1177

else
  cd /etc/X11/
  #download a dummy xorg.conf,
  #This is used by instances without GPU
  wget -q https://xpra.org/xorg.conf 
fi

#set the system to run the graphical mode (init 5)
systemctl set-default graphical.target
#and start GDM
systemctl enable gdm.service
systemctl start gdm.service
sleep 2

#enable and start also DCV
systemctl enable dcvserver.service
systemctl start dcvserver.service
sleep 2

#create a NICE DCV session
dcv create-session --owner centos --user centos test1
echo "centos:<YourPassword>" | chpasswd

;;
ComputeFleet)

#nothing here, for now


;;
esac

[ebs shared-fs]
volume_size = <size in GB>
shared_dir = /shared
#ebs_snapshot_id =
volume_type = io1
volume_iops = <number of IOPS>

#fsx_settings = parallel-fs

ebs_settings = shared-fs

aws s3 cp fluent-post-install.sh s3://<Your-S3-Bucket>/fluent-post-install.sh

pcluster create -c fluent.config fluent-test -t fluent_cluster -r <your-preferred-region>

#!/bin/bash

#check the installation directory
if [ ! -d "${1}" -o -z "${1}" ]; then
 echo "Error: please check the install dir"
 exit -1
fi

ansysDir="${1}/ansys_inc"
installDir="${1}/"

ansysDisk1="ANSYS2019R2_LINX64_Disk1.iso"
ansysDisk2="ANSYS2019R2_LINX64_Disk2.iso"

# mount the Disks
disk1="${installDir}/AnsysDisk1"
disk2="${installDir}/AnsysDisk2"
mkdir -p "${disk1}"
mkdir -p "${disk2}"

echo "Mounting ${ansysDisk1} ..."
mount -o loop "${installDir}/${ansysDisk1}" "${disk1}"

echo "Mounting ${ansysDisk2} ..."
mount -o loop "${installDir}/${ansysDisk2}" "${disk2}"

# INSTALL Ansys WB
echo "Installing Ansys ${ansysver}"
"${disk1}/INSTALL" -silent -install_dir "${ansysDir}" -media_dir2 "${disk2}"

echo "Ansys installed"

umount -l "${disk1}"
echo "${ansysDisk1} unmounted..."

umount -l "${disk2}"
echo "${ansysDisk2} unmounted..."

echo "Cleaning up temporary install directory"
rm -rf "${disk1}"
rm -rf "${disk2}"

echo "Installation process completed!"

aws ec2 create-snapshot --volume-id vol-xyz --description "This is the snapshot of my Ansys installation"

ebs_snapshot_id = snap-XXXX

#!/bin/bash

#$ -N Fluent
#$ -j Y
#$ -S /bin/bash

export ANSYSLI_SERVERS=2325@<your-license-server>
export ANSYSLMD_LICENSE_FILE=1055@<your-license-server>

basedir="/fsx"
workdir="${basedir}/$(date "+%d-%m-%Y-%H-%M")-${NSLOTS}-$RANDOM"
mkdir "${workdir}"
cd "${workdir}"
cp "${basedir}/f1_racecar_140m.tar.gz" .
tar xzvf f1_racecar_140m.tar.gz
rm -f f1_racecar_140m.tar.gz
cd bench/fluent/v6/f1_racecar_140m/cas_dat

${basedir}/ansys_inc/v194/fluent/bin/fluentbench.pl f1_racecar_140m -t${NSLOTS} -cnf=$PE_HOSTFILE -part=1 -nosyslog -noloadchk -ssh -mpi=intel -cflush