Amazon EKS のコンテナ起動時間を Bottlerocket のデータボリュームを活用することで短縮

cd ~/environment
git clone https://github.com/aws-samples/containers-blog-maelstrom.git 
cd containers-blog-maelstrom/bottlerocket-images-cache
find . -name "*.sh" | xargs chmod +755

cd ~/environment/containers-blog-maelstrom/bottlerocket-images-cache
./cloud9_init.sh

eksctl version
aws sts get-caller-identity --query Arn | grep eksworkshop-admin -q && echo "IAM role valid" || echo "IAM role NOT valid"

export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account)
export AWS_REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r '.region')
export AZS=($(aws ec2 describe-availability-zones --query 'AvailabilityZones[].ZoneName' --output text --region $AWS_REGION))
export EKS_CLUSTER_NAME=bottlerocket-eks-simulation

test -n "$AWS_REGION" && echo AWS_REGION is "$AWS_REGION" || echo AWS_REGION is not set

echo "export ACCOUNT_ID=${ACCOUNT_ID}" | tee -a ~/.bash_profile
echo "export AWS_REGION=${AWS_REGION}" | tee -a ~/.bash_profile
echo "export AZS=(${AZS[@]})" | tee -a ~/.bash_profile
echo "export AZS=(${AZS[@]})" | tee -a ~/.bash_profile
echo "export EKS_CLUSTER_NAME=${EKS_CLUSTER_NAME}" | tee -a ~/.bash_profile
aws configure set default.region ${AWS_REGION}
aws configure get default.region

cd ~/environment/containers-blog-maelstrom/bottlerocket-images-cache
./snapshot.sh -r $AWS_REGION public.ecr.aws/eks-distro/kubernetes/pause:3.2,public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2

[1/8] Deploying EC2 CFN stack ...

Waiting for changeset to be created..
Waiting for stack create/update to complete
Successfully created/updated stack - Bottlerocket-ebs-snapshot
[2/8] Launching SSM . done!
[3/8] Stopping kubelet.service .. done!
[4/8] Cleanup existing images .. done!
[5/8] Pulling ECR images:
  public.ecr.aws/eks-distro/kubernetes/pause:3.2 - amd64 ... done
  public.ecr.aws/eks-distro/kubernetes/pause:3.2 - arm64 ... done
  public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2 - arm64 ... done
 done!
[6/8] Stopping instance ... done!
[7/8] Creating snapshot ... done!
[8/8] Cleanup.
--------------------------------------------------
All done! Created snapshot in ap-northeast-1: snap-xxxxxxxxx

EBS_SNAPSHOT_ID=snap-xxxxxxx
echo "export EBS_SNAPSHOT_ID=${EBS_SNAPSHOT_ID}" | tee -a ~/.bash_profile

cd ~/environment/containers-blog-maelstrom/bottlerocket-images-cache
cat <<EOF> eks-clusterconfig.yaml
---
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  name: $EKS_CLUSTER_NAME
  region: $AWS_REGION
  version: '1.24'
iam:
  withOIDC: true
managedNodeGroups:
  - name: no-prefetch-mng
    instanceType: m5.2xlarge
    desiredCapacity: 1
    minSize: 1
    maxSize: 2
    privateNetworking: true
    amiFamily: Bottlerocket
  - name: prefetch-mng
    instanceType: m5.2xlarge
    desiredCapacity: 1
    minSize: 1
    maxSize: 2
    privateNetworking: true
    amiFamily: Bottlerocket
    additionalVolumes:
      - volumeName: '/dev/xvda' # OS Volume
      - volumeName: '/dev/xvdb'  # Data Volume
        snapshotID: $EBS_SNAPSHOT_ID
EOF

eksctl create cluster -f eks-clusterconfig.yaml

aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION

watch kubectl get nodes

Every 2.0s: kubectl get nodes -o wide 

NAME                                                 STATUS   ROLES    AGE     VERSION                INTERNAL-IP       EXTERNAL-IP   OS-IMAGE                                KERNEL-VERSION   CONTAINER-RUNTIME      
ip-192-168-xxx-xxx.ap-northeast-1.compute.internal   Ready    <none>   2m36s   v1.24.15-eks-fae4244   192.168.xxx.xxx   <none>        Bottlerocket OS 1.14.2 (aws-k8s-1.24)   5.15.117         containerd://1.6.20+bot
tlerocket                                                                                                                                                                                                             
ip-192-168-xxx-xxx.ap-northeast-1.compute.internal   Ready    <none>   2m30s   v1.24.15-eks-fae4244   192.168.xxx.xxx   <none>        Bottlerocket OS 1.14.2 (aws-k8s-1.24)   5.15.117         containerd://1.6.20+bot
tlerocket 

kubectl apply -f - << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: inflate-no-prefetch
spec:
  replicas: 1
  selector:
    matchLabels:
      app: inflate-no-prefetch
  template:
    metadata:
      labels:
        app: inflate-no-prefetch
    spec:
      terminationGracePeriodSeconds: 0
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                - key: "alpha.eksctl.io/nodegroup-name"
                  operator: "In"
                  values: ["no-prefetch-mng"]
      containers:
        - name: inflate-amazon-linux
          image: public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2
          resources:
            requests:
              cpu: 1
EOF

kubectl apply -f - << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: inflate-prefetch
spec:
  replicas: 1
  selector:
    matchLabels:
      app: inflate-prefetch
  template:
    metadata:
      labels:
        app: inflate-prefetch
    spec:
      terminationGracePeriodSeconds: 0
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                - key: "alpha.eksctl.io/nodegroup-name"
                  operator: "In"
                  values: ["prefetch-mng"]
      containers:
        - name: inflate-bottlerocket
          image: public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2
          resources:
            requests:
              cpu: 1
EOF

NO_PREFETCH_POD=$(kubectl get pod -l app=inflate-no-prefetch -o jsonpath="{.items[0].metadata.name}")
kubectl get events -o custom-columns=Time:.lastTimestamp,From:.source.component,Type:.type,Reason:.reason,Message:.message  --field-selector involvedObject.name=$NO_PREFETCH_POD,involvedObject.kind=Pod

Time                   From                Type     Reason      Message
2023-08-07T17:13:52Z   default-scheduler   Normal   Scheduled   Successfully assigned default/inflate-no-prefetch-6c45cd9b4c-c7dn8 to ip-192-168-180-250.ap-northeast-1.compute.internal
2023-08-07T17:13:53Z   kubelet             Normal   Pulling     Pulling image "public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2"
2023-08-07T17:14:41Z   kubelet             Normal   Pulled      Successfully pulled image "public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2" in 48.003125227s
2023-08-07T17:14:41Z   kubelet             Normal   Created     Created container inflate-amazon-linux
2023-08-07T17:14:41Z   kubelet             Normal   Started     Started container inflate-amazon-linux

PREFETCH_POD=$(kubectl get pod -l app=inflate-prefetch -o jsonpath="{.items[0].metadata.name}")
kubectl get events -o custom-columns=Time:.lastTimestamp,From:.source.component,Type:.type,Reason:.reason,Message:.message  --field-selector involvedObject.name=$PREFETCH_POD,involvedObject.kind=Pod

Time                   From                Type     Reason      Message
2023-08-07T17:13:53Z   default-scheduler   Normal   Scheduled   Successfully assigned default/inflate-prefetch-f5ff79858-g87cb to ip-192-168-97-187.ap-northeast-1.compute.internal
2023-08-07T17:13:54Z   kubelet             Normal   Pulled      Container image "public.ecr.aws/kubeflow-on-aws/notebook-servers/jupyter-pytorch:1.12.1-cpu-py38-ubuntu20.04-ec2-v1.2" already present on machine
2023-08-07T17:13:55Z   kubelet             Normal   Created     Created container inflate-bottlerocket
2023-08-07T17:13:56Z   kubelet             Normal   Started     Started container inflate-bottlerocket

apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: bottlerocket-provisioner
spec:
  providerRef:
    name: bottlerocket
  labels:
    billing-team: map-team
  annotations:
    example.com/owner: "my-team"
  requirements:
    - key: "node.kubernetes.io/instance-type"
      operator: In
      values: ["m5.2xlarge"]
    - key: "karpenter.sh/capacity-type" 
      operator: In
      values: [ "spot", "on-demand" ]

  limits:
    resources:
      cpu: 1000

  ttlSecondsAfterEmpty: 30

  weight: 20
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
  name: bottlerocket
spec:
  subnetSelector:
    karpenter.sh/discovery: xxxxxxxx-stack
  securityGroupSelector:
    karpenter.sh/discovery: xxxxxxxx-stack
  amiFamily: Bottlerocket
  tags:
    managed-by: "karpenter"
    intent: "api-server"
  blockDeviceMappings:
    - deviceName: /dev/xvda
      ebs:
        volumeSize: 10Gi
        volumeType: gp3
    - deviceName: /dev/xvdb
      ebs:
        volumeSize: 80Gi
        volumeType: gp3
        snapshotID: snap-xxxxxxxxxx

name: Build LLM Model and EBS Snapshot

on:
  push:
    branches: [ master ]
    paths:
      - 'sd-gen-sdkxl-consumer/**'

jobs:
  build_llm_image:
    runs-on: ubuntu-latest
    steps:
      // ...
      // skip build steps for docker login, docker build, docker push
      // for the complete script, please reference GitHub repo
      
  build_ebs:
    runs-on: ubuntu-latest
    # Only Run after `build_llm_image` job success (image have been pushed to ECR)
    needs: build_llm_image
    steps:
      - name: Checkout source code
        uses: actions/checkout@v2

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: YOUR_AWS_REGION

      - name: Build EBS Snapshot
        id: build-ebs-snapshot
        env:
          SNAPSHOT_ID: ""
        run: |
          cd sd-gen-sdkxl-consumer/automation
          . ./../run.sh
          echo "SNAPSHOT_ID:" $SNAPSHOT_ID

      - name: Commit and Push NodeTemplate
        id: update-node-template
        run: |
          echo "commit and push now"
          echo "SNAPSHOT_ID" $SNAPSHOT_ID

kubectl delete deploy inflate-no-prefetch --force
kubectl delete deploy inflate-prefetch --force

eksctl delete nodegroup no-prefetch-mng --cluster $EKS_CLUSTER_NAME 
eksctl delete nodegroup prefetch-mng --cluster $EKS_CLUSTER_NAME

eksctl delete cluster --name $EKS_CLUSTER_NAME

aws cloud9 list-environments

// Output:
{
    "environmentIds": [
        "5d36cbf9b1e54fa0af7aa80eef9bbb42"
    ]
}

aws cloud9 delete-environment --region $AWS_REGION --environment-id YOUR_CLOUD9_ENV_ID

snapshot.sh.
aws ec2 delete-snapshot --snapshot-id $EBS_SNAPSHOT_ID