AWS Batch および Amazon SageMaker を使用してオンラインの化合物溶解度予測ワークフローを構築する

docker pull amazonlinux:latest

docker run -it amazonlinux:latest
yum install epel-release aws-cli python-pip
pip install boto3

yum install python2-rdkit
mkdir -p /data

from __future__ import print_function
import os
import sys
import boto3
import argparse
import uuid
import datetime
import subprocess
from rdkit import Chem
import multiprocessing as mp
from multiprocessing import Pool,Process
from rdkit.Chem import Descriptors as desc

# establish an Amazon S3 connection; an IAM role is needed to access Amazon S3
s3=boto3.client('s3')
# change to execution dir
os.chdir("/data")

# execute the main RDKit descriptor worker
def smiles_desc(smiles_str):
    output_desc=[]
    m = Chem.MolFromSmiles(smiles_str)
    output_desc=smiles_str, \
           desc.MolWt(m), \
           desc.Ipc(m), \
           desc.TPSA(m), \
           desc.LabuteASA(m), \
           desc.NumHDonors(m), \
           desc.NumHAcceptors(m), \
           desc.MolLogP(m), \
           desc.HeavyAtomCount(m), \
           desc.NumRotatableBonds(m), \
           desc.RingCount(m), \
           desc.NumValenceElectrons(m)

    desc_shard=str(output_desc).strip('()')
    smiles_out(desc_shard,csv_header)

    return
# write out the results to the local filesystem
def smiles_out(single_des_out,csv_header):
    fd=open("%s_smiles_result.csv" %csv_header,"a")
    fd.write(single_des_out)
    fd.write('\n')
    fd.close
    
# upload to an s3 bucket
def s3_upload(csv_header):
    env_S3_OUT=os.environ['OUTPUT_SMILES_S3']
    s3.upload_file("%s_smiles_result.csv" %csv_header,"%s" %env_S3_OUT,"%s_smiles_result.csv" %csv_header)

# calculate stat function
def calc_perf(input_file,timedelta):
    calc_perf_stat=int(len(input_file)/timedelta.total_seconds())
    return calc_perf_stat

if __name__ == "__main__":
# accepted args for command line execution
   parser = argparse.ArgumentParser(description='Process SMILES')
   parser.add_argument("-i", dest="smiles", required=False, help='SMILES string')
   parser.add_argument("-f", dest="smiles_file", required=False, help="File for Batch SMILES processing")
   args = parser.parse_args()

   smiles_str=args.smiles
   smiles_files=args.smiles_file
   csv_header=str(uuid.uuid4())
   
# if defined on the command line
   if smiles_str:
      print (smiles_desc(smiles_str))
      smiles_out(str(smiles_desc(smiles_str)),csv_header)
      s3_upload(csv_header)
      
# if importing a file
# by default, parallelism expands to available cores exposed to the container
   elif smiles_files:
      infile=open(smiles_files,"r")
      pool = mp.Pool()
      start_calc=datetime.datetime.now()
      smiles_list=list(infile)
      pool.map(smiles_desc,smiles_list)
      pool.close()
      end_calc=datetime.datetime.now()
      delta_calc=end_calc-start_calc
      stat_calc=calc_perf(smiles_list,delta_calc)
      print ("number of structures/sec: %s"%stat_calc)
      s3_upload(csv_header)
      
# if file is defined through env
# by default, parallelism expands to available cores exposed to the container
   elif os.getenv('INPUT_SMILES'):
      env_SMILES=os.environ['INPUT_SMILES']
      infile=open(env_SMILES,"r")
      pool = mp.Pool()
      start_calc=datetime.datetime.now()
      smiles_list=list(infile)
      pool.map(smiles_desc,smiles_list)
      pool.close()
      end_calc=datetime.datetime.now()
      delta_calc=end_calc-start_calc
      stat_calc=calc_perf(smiles_list,delta_calc)
      print ("number of structures/sec: %s"%stat_calc)
      s3_upload(csv_header)

# if file is in the S3 import bucket, specify an AWS BATCH job def
# by default, parallelism expands to available cores exposed to the container
   elif os.getenv('INPUT_SMILES_S3'):
      env_S3_SMILES=os.environ['INPUT_SMILES_S3']
      s3_dn=subprocess.Popen("aws s3 cp %s /data" %env_S3_SMILES, shell=True)
      s3_dn.communicate()
      s3_file=env_S3_SMILES.split('/')[-1]
      infile=open("/data/%s" %s3_file,"r")
      pool = mp.Pool()
      start_calc=datetime.datetime.now()
      smiles_list=list(infile)
      pool.map(smiles_desc,smiles_list)
      pool.close()
      end_calc=datetime.datetime.now()
      delta_calc=end_calc-start_calc
      stat_calc=calc_perf(smiles_list,delta_calc)
      print ("number of structures/sec: %s"%stat_calc)
      s3_upload(csv_header)
      
   else:
      print ("No SMILES FOUND...")
      sys.exit(0)

<account_number>.dkr.ecr.us-east-1.amazonaws.com/awsbatch/rdkit

# Get the temporary login credentials of your registry.
aws ecr get-login —no-include-email 

# Execute the login command from "aws ecr get-login".
docker login <params>

# Commit the local container to a new docker image.
docker commit -m "setting up rdkit infra" <containerid> amazonlinux:rdkit

# Tag the new image with your Amazon ECR tag created in the Amazon ECS console.
docker tag amazonlinux:rdkit <account_number>.dkr.ecr.us-east-1.amazonaws.com/awsbatch/rdkit:latest

# Push the image to your private registery.
docker push <account_number>.dkr.ecr.us-east-1.amazonaws.com/awsbatch/rdkit:latest

#SET ENVIROMENT VARIABLE

#Set the bucket name for the processed bucket from AWS-Batch
#Set the key name for the output csv file from AWS-Batch

BUCKET_NAME="rdkit-processed"
KEY_NAME="final_sol.csv"

#Install RDKit through conda
!conda install -y -c rdkit rdkit

import os
import sys
import csv
import urllib
import boto
import subprocess
import numpy as np
import mxnet as mx
from rdkit import Chem
from rdkit.Chem import Draw
import multiprocessing as mp
import matplotlib.pyplot as plt
from ipywidgets import widgets
from IPython.html.widgets import *
from matplotlib.gridspec import GridSpec
from rdkit.Chem.Draw import IPythonConsole
from multiprocessing import Pool,Process
from rdkit.Chem import Descriptors as desc

# Download File
urllib.urlretrieve ("https://raw.githubusercontent.com/deepchem/deepchem/master/datasets/delaney-processed.csv", \
                    "delaney-processed.csv")

# Import File
infile=open("delaney-processed.csv","rb")
reader = csv.DictReader(infile)
smiles_list=list(reader)
m=[]
for i in range(len(smiles_list)):
    m.append(Chem.MolFromSmiles(smiles_list[i]['smiles']))
print "Number of Structures in file: %s" %len(m)

#Draw a few structures
Draw.MolsToGridImage(m[:8],molsPerRow=4,subImgSize=(200,200),legends=[smiles_list[i]['Compound ID'] \
                                                                      for i in xrange(len(m[:8]))])

# Import our result csv file from the AWS Batch workflow
conn=boto.connect_s3()
bucket=conn.get_bucket(BUCKET_NAME)
key=bucket.get_key(KEY_NAME)
key.get_contents_to_filename(KEY_NAME)

batch_file=open(KEY_NAME,"rb")
batch_reader = csv.reader(batch_file)
out_desc_all=list(batch_reader)
out_desc = [x[1:13] for x in out_desc_all]

# Define our input data arrays
out_desc_array=np.array(out_desc,dtype=np.float32)
np.random.shuffle(out_desc_array)
all_data = out_desc_array[:,0:11]
all_label = out_desc_array[:,11:12]
x_train,x_label, y_test,y_label = out_desc_array[:790,0:11], \
                                  out_desc_array[:790,11:12], \
                                  out_desc_array[790:,0:11], \
                                  out_desc_array[790:,11:12]
            
print np.shape(all_data)
print np.shape(all_label)
print np.shape(x_train)
print np.shape(x_label)
print np.shape(y_test)
print np.shape(y_label)

# Define Model Parameters
import logging
logging.getLogger().setLevel(logging.DEBUG)

batch_size=20
train_iter = mx.io.NDArrayIter(x_train,x_label, batch_size, shuffle=True,label_name='lin_reg_label')
eval_iter = mx.io.NDArrayIter(y_test, y_label, batch_size, shuffle=False)
all_iter = mx.io.NDArrayIter(all_data, all_label, batch_size, shuffle=False)

train_iter.__dict__

X = mx.sym.Variable('data')
Y = mx.symbol.Variable('lin_reg_label')
fc1 = mx.symbol.BatchNorm_v1(data=X, name="BatchNorm_v1")
tanh1 = mx.symbol.Activation(fc1, act_type="tanh", name="tanh1")
fc2 = mx.sym.FullyConnected(data=tanh1, name='fc2', num_hidden = 1)
lro = mx.sym.LinearRegressionOutput(data=fc2, label=Y, name="lro")
shape={"data": (batch_size,790, 11)}
mx.viz.plot_network(lro,shape = shape)

# Train our model
model = mx.mod.Module(
    symbol = lro ,
    context = mx.cpu(),
    data_names=['data'],
    label_names = ['lin_reg_label']# network structure
)
model.bind(data_shapes=[train_iter.provide_data[0]], label_shapes=[train_iter.provide_label[0]])
model.init_params()
model.fit(train_iter, eval_iter,
            optimizer='sgd',
            optimizer_params={'learning_rate':0.01, 'momentum':0.9},
            num_epoch=50,
            eval_metric='rmse',
            batch_end_callback = mx.callback.Speedometer(batch_size, 10))

# Plot our metrics - full cross validation
%matplotlib inline
from scipy import stats
all_preds=model.predict(all_iter).asnumpy()
eval_preds=model.predict(eval_iter).asnumpy()

plt.figure(figsize=(6,6), dpi=80)
plt.scatter(all_label,all_preds, label='All', c='blue',marker='.', alpha=0.2)
plt.scatter(y_label, eval_preds, label='Test/Evaluation', c='green',marker='.')
plt.xlim(-12,3)
plt.ylim(-10,2)
plt.title('Predicted vs Measured ESOLV DeepChem Set')
plt.xlabel('Measured logSolubility (mol/L)')
plt.ylabel('Predicted logSolubility (mol/L)')
plt.legend(loc=4) 

print "pearson coefficient of all data: %s" %round(float(stats.pearsonr(all_label,all_preds)[0]),3)
print "pearson coefficient of evaluation data: %s" %round(float(stats.pearsonr(y_label,eval_preds)[0]),3)

delta_preds=all_preds-all_label
def plthist(x):
    fig = plt.figure(figsize=(6,6), dpi=80)
    plt.hist(delta_preds,bins=x,align='mid',range=(-4,4),rwidth=0.8)
    x=plt.xlabel('Element-wise Evaluation Prediction-Target')
interact(plthist,x=(2,50,1))