Amazon ECS で Amazon Elastic Inference ワークロードを実行する

{
  "Version": "2008-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

{
    "Statement": [
        {
            "Effect": "Allow",
            "Resource": "*",
            "Action": [
                "elastic-inference:Connect",
                "iam:List*",
                "iam:Get*",
                "ec2:Describe*",
                "ec2:Get*"
            ]
        }
    ],
    "Version": "2012-10-17"
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": "ecs-tasks.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

# install unzip
apt-get --assume-yes install unzip
# download and unzip the model
wget https://s3-us-west-2.amazonaws.com/aws-tf-serving-ei-example/ssd_resnet.zip -P /models/ssdresnet/
unzip -j /models/ssdresnet/ssd_resnet.zip -d /models/ssdresnet/1
# download and extract Elastic Inference enabled TensorFlow Serving
wget https://s3.amazonaws.com/amazonei-tensorflow/tensorflow-serving/v1.13/ubuntu/latest/tensorflow-serving-1-13-1-ubuntu-ei-1-1.tar.gz
tar xzvf tensorflow-serving-1-13-1-ubuntu-ei-1-1.tar.gz
# make the binary executable
chmod +x tensorflow-serving-1-13-1-ubuntu-ei-1-1/amazonei_tensorflow_model_server
# Unset the ECS_CONTAINER_METADATA_URI environment variable to force Elastic Inference endpoint metadata lookup from ECS container instance's metadata.
# Otherwise, Elastic Inference endpoint metadata would tried to be retrieved from container metadata, which would fail.  
env -u ECS_CONTAINER_METADATA_URI tensorflow-serving-1-13-1-ubuntu-ei-1-1/amazonei_tensorflow_model_server --port=${GRPC_PORT} --rest_api_port=${REST_PORT} --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME}

#!/bin/bash 

tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} "$@"

{
    "executionRoleArn": "arn:aws:iam::<replace-with-your-account-id>:role/ecsTaskExecutionRole",
    "containerDefinitions": [
        {
            "entryPoint": [
                "bash",
                "-c",
                "apt-get --assume-yes install unzip; wget https://s3-us-west-2.amazonaws.com/aws-tf-serving-ei-example/ssd_resnet.zip -P ${MODEL_BASE_PATH}/${MODEL_NAME}/; unzip -j ${MODEL_BASE_PATH}/${MODEL_NAME}/ssd_resnet.zip -d ${MODEL_BASE_PATH}/${MODEL_NAME}/1/; /usr/bin/tf_serving_entrypoint.sh"
            ],
            "portMappings": [
                {
                    "hostPort": 8500,
                    "protocol": "tcp",
                    "containerPort": 8500
                },
                {
                    "hostPort": 8501,
                    "protocol": "tcp",
                    "containerPort": 8501
                }
            ],
            "cpu": 0,
            "environment": [
                {
                    "name": "KMP_SETTINGS",
                    "value": "0"
                },
                {
                    "name": "TENSORFLOW_INTRA_OP_PARALLELISM",
                    "value": "2"
                },
                {
                    "name": "MODEL_NAME",
                    "value": "ssdresnet"
                },
                {
                    "name": "KMP_AFFINITY",
                    "value": "granularity=fine,compact,1,0"
                },
                {
                    "name": "MODEL_BASE_PATH",
                    "value": "/models"
                },
                {
                    "name": "KMP_BLOCKTIME",
                    "value": "0"
                },
                {
                    "name": "TENSORFLOW_INTER_OP_PARALLELISM",
                    "value": "2"
                },
                {
                    "name": "OMP_NUM_THREADS",
                    "value": "1"
                }
            ],
            "mountPoints": [],
            "volumesFrom": [],
            "image": "763104351884.dkr.ecr.us-east-1.amazonaws.com/tensorflow-inference:1.13-cpu-py27-ubuntu16.04",
            "essential": true,
            "name": "ubuntu-tfs"
        },
        {
            "entryPoint": [
                "bash",
                "-c",
                "apt-get --assume-yes install unzip; wget https://s3-us-west-2.amazonaws.com/aws-tf-serving-ei-example/ssd_resnet.zip -P /models/ssdresnet/; unzip -j /models/ssdresnet/ssd_resnet.zip -d /models/ssdresnet/1; wget https://s3.amazonaws.com/amazonei-tensorflow/tensorflow-serving/v1.13/ubuntu/latest/tensorflow-serving-1-13-1-ubuntu-ei-1-1.tar.gz; tar xzvf tensorflow-serving-1-13-1-ubuntu-ei-1-1.tar.gz; chmod +x tensorflow-serving-1-13-1-ubuntu-ei-1-1/amazonei_tensorflow_model_server; env -u ECS_CONTAINER_METADATA_URI tensorflow-serving-1-13-1-ubuntu-ei-1-1/amazonei_tensorflow_model_server --port=${GRPC_PORT} --rest_api_port=${REST_PORT} --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME}"
            ],
            "portMappings": [
                {
                    "hostPort": 9000,
                    "protocol": "tcp",
                    "containerPort": 9000
                },
                {
                    "hostPort": 9001,
                    "protocol": "tcp",
                    "containerPort": 9001
                }
            ],
            "cpu": 0,
            "environment": [
                {
                    "name": "GRPC_PORT",
                    "value": "9000"
                },
                {
                    "name": "REST_PORT",
                    "value": "9001"
                },
                {
                    "name": "MODEL_NAME",
                    "value": "ssdresnet"
                },
                {
                    "name": "MODEL_BASE_PATH",
                    "value": "/models"
                }
            ],
            "mountPoints": [],
            "volumesFrom": [],
            "image": "763104351884.dkr.ecr.us-east-1.amazonaws.com/tensorflow-inference:1.13-cpu-py27-ubuntu16.04",
            "essential": true,
            "name": "ubuntu-tfs-ei"
        }
    ],
    "memory": "2048",
    "taskRoleArn": "arn:aws:iam::<replace-with-your-account-id>:role/ecs-ei-task-role",
    "family": "ei-ecs-ubuntu-tfs-bridge-s3",
    "requiresCompatibilities": [
        "EC2"
    ],
    "networkMode": "bridge",
    "volumes": [],
    "placementConstraints": []
}

curl -O https://raw.githubusercontent.com/amikelive/coco-labels/master/coco-labels-paper.txt
curl -O https://s3.amazonaws.com/amazonei/media/3giraffes.jpg

from __future__ import print_function
import grpc
import tensorflow as tf
from PIL import Image
import numpy as np
import time
import os
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc

tf.app.flags.DEFINE_string('server', 'localhost:8500',
                           'PredictionService host:port')
tf.app.flags.DEFINE_string('image', '', 'path to image in JPEG format') 
FLAGS = tf.app.flags.FLAGS

if(FLAGS.image == ''):
  print("Supply an Image using '--image [path/to/image]'")
  exit(1)

local_coco_classes_txt = "coco-labels-paper.txt"
 
# Setting default number of predictions
NUM_PREDICTIONS = 20

# Reading coco labels to a list 
with open(local_coco_classes_txt) as f:
  classes = ["No Class"] + [line.strip() for line in f.readlines()]

def main(_):
 
  channel = grpc.insecure_channel(FLAGS.server)
  stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
 
  with Image.open(FLAGS.image) as f:
    f.load()
    
    # Reading the test image given by the user 
    data = np.asarray(f)

    # Setting batch size to 1
    data = np.expand_dims(data, axis=0)

    # Creating a prediction request 
    request = predict_pb2.PredictRequest()
 
    # Setting the model spec name
    request.model_spec.name = 'ssdresnet'
 
    # Setting up the inputs and tensors from image data
    request.inputs['inputs'].CopyFrom(
        tf.contrib.util.make_tensor_proto(data, shape=data.shape))
 
    # Iterating over the predictions.The first inference request can take several seconds to complete

    durations = []

    for curpred in range(NUM_PREDICTIONS): 
      if(curpred == 0):
        print("The first inference request loads the model into the accelerator and can take several seconds to complete.Please standby!")

      # Start the timer 
      start = time.time()
 
      # This is where the inference actually happens 
      result = stub.Predict(request, 60.0)  # 60 secs timeout
      duration = time.time() - start
      durations.append(duration)
      print("Inference %d took %f seconds" % (curpred, duration))

    # Extracting results from output 
    outputs = result.outputs
    detection_classes = outputs["detection_classes"]

    # Creating an ndarray from the output TensorProto
    detection_classes = tf.make_ndarray(detection_classes)

    # Creating an ndarray from the detection_scores
    detection_scores = tf.make_ndarray(outputs['detection_scores'])
 
    # Getting the number of objects detected in the input image from the output of the predictor 
    num_detections = int(tf.make_ndarray(outputs["num_detections"])[0])
    print("%d detection[s]" % (num_detections))

    # Getting the class ids from the output and mapping the class ids to class names from the coco labels with associated detection score
    class_label_score = ["%s: %.2f" % (classes[int(detection_classes[0][index])], detection_scores[0][index]) 
                   for index in range(num_detections)]
    print("SSD Prediction is (label, probability): ", class_label_score)
    print("Latency:")
    for percentile in [95, 50]:
      print("p%d: %.2f seconds" % (percentile, np.percentile(durations, percentile, interpolation='lower')))
 
if __name__ == '__main__':
  tf.app.run()

SERVER_IP=<replace-with-ECS-container-instance-IP-address>
for PORT in 8501 9001
do
  curl -s http://${SERVER_IP}:${PORT}/v1/models/ssdresnet
done

{
 "model_version_status": [
  {
   "version": "1",
   "state": "AVAILABLE",
   "status": {
    "error_code": "OK",
    "error_message": ""
   }
  }
 ]
}

source activate amazonei_tensorflow_p27
for PORT in 8500 9000
do
  python ssd_resnet_client.py --server=${SERVER_IP}:${PORT} --image 3giraffes.jpg
done

The first inference request loads the model into the accelerator and can take several seconds to complete.Please standby!
Inference 0 took 12.923095 seconds
Inference 1 took 1.363095 seconds
Inference 2 took 1.338855 seconds
Inference 3 took 1.311022 seconds
Inference 4 took 1.305457 seconds
Inference 5 took 1.303680 seconds
Inference 6 took 1.297357 seconds
Inference 7 took 1.302721 seconds
Inference 8 took 1.299495 seconds
Inference 9 took 1.293291 seconds
Inference 10 took 1.305852 seconds
Inference 11 took 1.292999 seconds
Inference 12 took 1.300874 seconds
Inference 13 took 1.300001 seconds
Inference 14 took 1.297276 seconds
Inference 15 took 1.297859 seconds
Inference 16 took 1.305029 seconds
Inference 17 took 1.315366 seconds
Inference 18 took 1.288984 seconds
Inference 19 took 1.289530 seconds
4 detection[s]
SSD Prediction is (label, probability):  ['giraffe: 0.84', 'giraffe: 0.74', 'giraffe: 0.68', 'giraffe: 0.50']
Latency:
p95: 1.36 seconds
p50: 1.30 seconds
The first inference request loads the model into the accelerator and can take several seconds to complete.Please standby!
Inference 0 took 14.081767 seconds
Inference 1 took 0.295794 seconds
Inference 2 took 0.293941 seconds
Inference 3 took 0.311396 seconds
Inference 4 took 0.291605 seconds
Inference 5 took 0.285228 seconds
Inference 6 took 0.226951 seconds
Inference 7 took 0.283834 seconds
Inference 8 took 0.290349 seconds
Inference 9 took 0.228826 seconds
Inference 10 took 0.284496 seconds
Inference 11 took 0.293179 seconds
Inference 12 took 0.296765 seconds
Inference 13 took 0.230531 seconds
Inference 14 took 0.283406 seconds
Inference 15 took 0.292458 seconds
Inference 16 took 0.300849 seconds
Inference 17 took 0.294651 seconds
Inference 18 took 0.293372 seconds
Inference 19 took 0.225444 seconds
4 detection[s]
SSD Prediction is (label, probability):  ['giraffe: 0.84', 'giraffe: 0.74', 'giraffe: 0.68', 'giraffe: 0.50']
Latency:
p95: 0.31 seconds
p50: 0.29 seconds