基于 NONMEM、PsN 和 Amazon ParallelCluster 服务构建云端药代动力学/药效学建模及数据分析流程

(pcluster3) [ec2-user@ip-172-16-0-97 ~]$ pcluster list-official-images --os ubuntu2004 --architecture x86_64
{
  "images": [
    {
      "amiId": "ami-0c1c0f2403b85803e",
      "os": "ubuntu2004",
      "name": "aws-parallelcluster-3.5.1-ubuntu-2004-lts-hvm-x86_64-202303171149 2023-03-17T11-53-21.883Z",
      "version": "3.5.1",
      "architecture": "x86_64"
    }
  ]
}

sudo apt install gcc-9
sudo cp /usr/bin/gcc-9 /usr/bin/gcc
sudo apt install unzip
sudo apt install gfortran-9
sudo cp /usr/bin/gfortran-9 /usr/bin/gfortran
mkdir nonmem # nonmem 安装包解压放到此目录
mkdir nm751 # nonmem 7.5.1 的安装目录

ubuntu@ip-172-16-0-240:~/nonmem/nm751CD$ /bin/bash SETUP75 ~/nonmem/nm751CD/ ~/nm751 gfortran

sudo apt install python3 python3-venv python3-dev
sudo apt install make
sudo apt install r-base pandoc libpq-dev libcairo2-dev libssl-dev libcurl4-openssl-dev libmariadb-dev libgmp-dev libmpfr-dev libxml2-dev libudunits2-dev libblas-dev liblapack-dev libmagick++-dev

root@ip-172-16-0-208:~# perl -MCPAN -e shell
Terminal does not support AddHistory.

To fix enter>  install Term::ReadLine::Perl

cpan shell -- CPAN exploration and modules installation (v2.22)
Enter 'h' for help.

cpan[1]> o conf urllist http://mirrors.tuna.tsinghua.edu.cn/CPAN/
Please use 'o conf commit' to make the config permanent!

cpan[2]> o conf commit
commit: wrote '/root/.local/share/.cpan/CPAN/MyConfig.pm'

cpan[3]> exit

sudo su
cpan Math::Random
cpan MouseX::Params::Validate
cpan Archive::Zip

ubuntu@ip-172-16-0-240:~/PsN/PsN-Source$ sudo perl setup.pl
This is the PsN installer. I will install PsN version 5.3.0.
You need to answer a few questions. If a default value is presented
you may accept it by pressing ENTER.

PsN Utilities installation directory [/usr/local/bin]:
Path to perl binary used to run Utilities [/usr/bin/perl]:
PsN Core and Toolkit installation directory [/usr/local/share/perl/5.30.0]:

The next step is to check Perl module dependencies.
If a module is missing, you must install it, e.g. from CPAN,
http://www.cpan.org/modules/index.html
before PsN can be run.

Would you like this script to check Perl modules [y/n]?y

Testing required modules:
Module Math::Random ok
Module MouseX::Params::Validate ok

Done testing required modules.

Testing recommended but not required modules...
Module Archive::Zip ok
Tests done.
Searching for NM versions on your file system...
No NM versions have been found. You need to add at least one.
Enter the *complete* path of the NM-installation directory: /home/ubuntu/nm751
These are the loaded NM versions:
/home/ubuntu/nm751
Would you like to add another one [y/n] ?
n
Enter the name you want to use for the NM-version in
/home/ubuntu/nm751
with PsN option -nm_version,
or press ENTER to use the name nm751 : 


A new configuration file has been created:
/usr/local/share/perl/5.30.0/PsN_5_3_0/psn.conf
If you want to add cluster options or personalized defaults or
NMqual settings you can do this manually by editing
/usr/local/share/perl/5.30.0/PsN_5_3_0/psn.conf
in a regular text editor.

Please note that if you have a psn.conf file in your home directory,
the settings in that file will override the settings in
/usr/local/share/perl/5.30.0/PsN_5_3_0/psn.conf

Installation complete.

Press ENTER to exit the installation program.

ubuntu@ip-172-16-0-240:~$ sudo -s
root@ip-172-16-0-240:/home/ubuntu# cd /usr/bin/
root@ip-172-16-0-240:/usr/bin# mkdir default.mpi
root@ip-172-16-0-240:/usr/bin# mv mpi* default.mpi
root@ip-172-16-0-240:/usr/bin# exit
ubuntu@ip-172-16-0-240:~/run010$ which mpirun
/opt/amazon/openmpi/bin//mpirun
ubuntu@ip-172-16-0-240:~/run010$ cd /opt/amazon/openmpi/bin/
ubuntu@ip-172-16-0-240:/opt/amazon/openmpi/bin$ sudo su
root@ip-172-16-0-240:/opt/amazon/openmpi/bin# mkdir default.mpi
root@ip-172-16-0-240:/opt/amazon/openmpi/bin# mv mpi* default.mpi
root@ip-172-16-0-240:/opt/amazon/openmpi/bin# exit

sudo su
apt remove mpich
wget https://github.com/pmodels/mpich/releases/download/v4.1.1/mpich-4.1.1.tar.gz
tar xzf mpich-4.1.1.tar.gz
cd mpich-4.1.1
./configure --prefix=/opt/mpich-4.1.1 CFLAGS="-m64" FFLAGS="-m64" --enable-f90 --enable-timer-type=gettimeofday --enable-static=yes 2>&1 | tee c.txt #--enable-static 生成静态库文件.a
make -j64 2>&1 | tee m.txt #-j 根据当前机器的 cpu 数而定，数值越大编译越快，建议 64 核以上
make install 2>&1 | tee mi.txt
cd /home/ubuntu/nm751/mpi/mpi_ling/
mv libmpich.a libmpich.a.bk
ln -s  /opt/mpich-4.1.1/lib/libmpi.a  /home/ubuntu/nm751/mpi/mpi_ling/libmpich.a  # 添加软连接到 nonmem 目录下的静态库
vim /etc/profile # 添加全局 mpich 环境变量
PATH=/opt/mpich-4.1.1/bin:$PATH; export PATH
source /etc/profile

ubuntu@ip-172-16-0-240:~/run010$ execute run010.ctl  -nodes=2 -parafile=/home/ubuntu/nm751/run/mpilinux8.pnm
Starting 1 NONMEM executions. 1 in parallel.
S:1 .. 
All executions started.
Starting NMTRAN
  
 WARNINGS AND ERRORS (IF ANY) FOR PROBLEM    1
             
 (WARNING  2) NM-TRAN INFERS THAT THE DATA ARE POPULATION.
  
Note: Analytical 2nd Derivatives are constructed in FSUBS but are never used.
      You may insert $ABBR DERIV2=NO after the first $PROB to save FSUBS construction and compilation time
  
Recompiling certain components

 USING PARALLEL PROFILE /home/ubuntu/nm751/run/mpilinux8.pnm
 MPI TRANSFER TYPE SELECTED
Building NONMEM Executable
Starting MPI version of nonmem execution ...
Manager Location ip-172-16-0-240//home/ubuntu/run010/modelfit_dir6/NM_run1
License Registered to: 
Expiration Date:    14 JUN 2023
Current Date:       17 MAY 2023
  **** WARNING!!! Days until program expires :  27 ****
  **** CONTACT idssoftware@iconplc.com FOR RENEWAL ****


 First Order Conditional Estimation with Interaction
 MONITORING OF SEARCH:


0ITERATION NO.:    0    OBJECTIVE VALUE:  -5434.19246879288        NO. OF FUNC. EVALS.:   9
 CUMULATIVE NO. OF FUNC. EVALS.:        9
 NPARAMETR:  1.0143E+01  9.9945E+00  1.0358E+00  1.0924E+02  4.8085E+01  9.8599E+00  4.7377E-02  5.7569E-02  2.1213E-02  4.2273E-03

 PARAMETER:  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01

 GRADIENT:   1.6906E+02  1.2308E+02  3.6609E+01  1.2369E+01  1.1555E+01  3.2891E+00  1.5793E-02 -9.3163E-04  5.3890E-02  1.2228E+00


0ITERATION NO.:    2    OBJECTIVE VALUE:  -5434.19246879288        NO. OF FUNC. EVALS.:  54
 CUMULATIVE NO. OF FUNC. EVALS.:       63
 NPARAMETR:  1.0143E+01  9.9945E+00  1.0358E+00  1.0924E+02  4.8085E+01  9.8599E+00  4.7377E-02  5.7569E-02  2.1213E-02  4.2273E-03

 PARAMETER:  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01  1.0000E-01

 GRADIENT:  -1.0877E+00  2.4267E-01 -3.0004E+00  1.6597E+00  8.3079E-01 -2.4949E+00  1.5682E-02 -1.0777E-03  5.4210E-02  9.4077E-01

 Elapsed estimation  time in seconds:     3.47
 Elapsed covariance  time in seconds:    10.12
 Elapsed postprocess time in seconds:     0.13
 Elapsed finaloutput time in seconds:     0.06
Done with nonmem execution
F:1 .. 
execute done

ubuntu@ip-172-16-0-240:~/run010$ bootstrap run010.ctl  -samples=30 -threads=2 -nodes=2 -parafile=/home/ubuntu/nm751/run/mpilinux8.pnm
Resampling from /home/ubuntu/run010/RUN010_NM.csv
|..............................| ... done
Starting 30 NONMEM executions. 2 in parallel.
S:1 .. S:4 .. S:7 .. S:10 .. S:13 .. S:16 .. S:19 .. S:22 .. S:25 .. S:28 .. S:30 .. Waiting for all NONMEM runs to finish:
F:30 .. F:29 ..  done
Bootstrap done.

sudo /usr/local/sbin/ami_cleanup.sh

Region: cn-northwest-1
Image:
  Os: ubuntu2004
  CustomAmi: ami-0ec6815c1a911172b
SharedStorage:
  - Name: efs
    MountDir: /efs
    StorageType: Efs
    EfsSettings:
      FileSystemId: fs-08b035afccad28cdb
HeadNode:
  InstanceType: c6i.xlarge
  Dcv:
    Enabled: true
  Networking:
    SubnetId: subnet-005c6ee6d3fd9ec03
    ElasticIp: true
  Ssh:
    KeyName: NONMEM
Scheduling:
  Scheduler: slurm
  SlurmQueues:
  - Name: thinnode
    ComputeResources:
    - Name: c6i8xlarge
      DisableSimultaneousMultithreading: false
      Instances:
      - InstanceType: c6i.8xlarge
      MinCount: 0
      MaxCount: 15
    Networking:
      SubnetIds:
      - subnet-060134b58cca98d98
  - Name: fatnode
    ComputeResources:
    - Name: c6imetal
      Instances:
      - InstanceType: c6i.metal
      MinCount: 0
      MaxCount: 3
    Networking:
      SubnetIds:
      - subnet-060134b58cca98d98
Monitoring:
  DetailedMonitoring: true
  Logs:
    CloudWatch:
      Enabled: true
      RetentionInDays: 30
      DeletionPolicy:  Delete
  Dashboards:
    CloudWatch:
      Enabled: true

 pcluster create-cluster -n <cluster_name> -c <configfile> --dryrun true # 通过 dryrun 命令验证集群配置文件的正确性
pcluster create-cluster -n <cluster_name> -c <configfile> -nr # 去掉 dryrun 正式创建集群，添加-nr 参数防止失败回滚方便troubleshooting

#!/bin/bash
/home/ubuntu/nm751/run/nmfe75 run010.ctl run010.lst -parafile=/home/ubuntu/nm751/run/mpilinux8.pnm [nodes]=4

sbatch --cpus-per-task=4 test1.sh

execute run010.ctl -nodes=4 -parafile=/home/ubuntu/nm751/run/mpilinux8.pnm -run_on_slurm -slurm_prepend_flags="--cpus-per-task=4"

bootstrap run010.ctl -samples=100 -threads=10 -nodes=4 -parafile=/home/ubuntu/nm751/run/mpilinux8.pnm -run_on_slurm -slurm_prepend_flags="--cpus-per-task=4"