Amazon EKS のスケジューリングをカスタマイズする

  annotations:
    custom-pod-schedule-strategy: 'label1Key=label1Value,base=1,weight=0:label2Key=label2Value,weight=1:label3Key=label3Value,weight=1'

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  namespace: test
  annotations:
    custom-pod-schedule-strategy: 'label1Key=label1Value,base=2,weight=1:label2Key=label2Value,weight=3'
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 10
  ....

export AWS_REGION=us-east-1 
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' --output text)
export CLUSTER_NAME=eks-custom-pod-schedule  # choose an existing EKS Cluster name
export ECR_REPO=custom-kube-scheduler-webhook
export SERVICE=custom-kube-scheduler-webhook
export NAMESPACE=custom-kube-scheduler-webhook
export SECRET=custom-kube-scheduler-webhook-certs

eksctl create cluster \
  --name $CLUSTER_NAME \
  --region $AWS_REGION \
  --version 1.21  \
  --managed

git clone https://github.com/aws-samples/containers-blog-maelstrom.git
cd custom-kubernetes-scheduler

IMAGE_REPO="${ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
IMAGE_NAME=${ECR_REPO}

export ECR_REPO_URI=$(aws ecr describe-repositories --repository-name ${IMAGE_NAME}  | jq -r '.repositories[0].repositoryUri')

if [ -z "$ECR_REPO_URI" ]
then
      echo "${IMAGE_REPO}/${IMAGE_NAME} does not exist. So creating it..."
      ECR_REPO_URI=$(aws ecr create-repository \
        --repository-name $IMAGE_NAME\
        --region $AWS_REGION \
        --query 'repository.repositoryUri' \
        --output text)
      echo "ECR_REPO_URI=$ECR_REPO_URI"
else
      echo "${IMAGE_REPO}/${IMAGE_NAME} already exist..."
fi

make

Building the custom-kube-scheduler-webhook binary for Docker (linux)...
Building the docker image: custom-kube-scheduler-webhook:latest...
Sending build context to Docker daemon  262.5MB
Step 1/6 : FROM alpine:latest
....
Successfully built bc4560ae8770
Successfully tagged XXXXXXX.dkr.ecr.us-east-1.amazonaws.com/custom-kube-scheduler-webhook:latest
Pushing the docker image for XXXXXXX.dkr.ecr.us-east-1.amazonaws.com/custom-kube-scheduler-webhook:latest ...
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin XXXXXXX.dkr.ecr.us-east-1.amazonaws.com/custom-kube-scheduler-webhook
WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded
The push refers to repository [XXXXXXX.dkr.ecr.us-east-1.amazonaws.com/custom-kube-scheduler-webhook]
d7eaac728432: Pushed 
b2d5eeeaba3a: Layer already exists 
latest: digest: sha256:5d65e7f5578d95221e7691cdb8415a8a80b1c9c553684bea3011df39262cbe4d size: 740

kubectl create ns $NAMESPACE

./deploy/webhook-create-signed-cert.sh \
    --service $SERVICE \
    --secret $SECRET \
    --namespace $NAMESPACE

kubectl get secret $SECRET -n $NAMESPACE -o json

export WEBHOOK_CONFIG="deploy/custom-kube-scheduler-webhook-config.yaml"
cat deploy/custom-kube-scheduler-webhook-config-template.yaml | \
    deploy/webhook-patch-ca-bundle.sh >  $WEBHOOK_CONFIG
kubectl apply -f $WEBHOOK_CONFIG

export WEBHOOK_CONTROLLER="deploy/custom-kube-scheduler-webhook-controller.yaml"
envsubst < deploy/custom-kube-scheduler-webhook-controller-template.yaml > $WEBHOOK_CONTROLLER
kubectl apply -f $WEBHOOK_CONTROLLER

kubectl apply -f - <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: test
  labels:
    custom-kube-scheduler-webhook: enabled 
---
EOF

kubectl apply -f - <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  namespace: test
  annotations:
    custom-pod-schedule-strategy: 'karpenter.sh/capacity-type=on-demand,base=2,weight=1:karpenter.sh/capacity-type=spot,weight=3'
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 10
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: public.ecr.aws/nginx/nginx:latest
        imagePullPolicy: Always
        name: nginx
        resources:
          limits:
            cpu:  400m
            memory: 1600Mi
          requests:
            cpu: 400m
            memory: 1600Mi        
        ports:
        - name: http
          containerPort: 80
---
EOF

kubectl logs -f -lapp=custom-kube-scheduler-webhook  -n custom-kube-scheduler-webhook         
I0110 13:05:45.187535       1 webhook.go:373] flow=CREATE serviceInstanceNum=67 Found a deployment nginx in namespace test with total replicas 10 and strategy=karpenter.sh/capacity-type=on-demand,base=2,weight=1:karpenter.sh/capacity-type=spot,weight=3
I0110 13:05:45.187561       1 webhook.go:378] flow=CREATE serviceInstanceNum=67 nodeLabelStrategyList=[{karpenter.sh/capacity-type=on-demand 4 1} {karpenter.sh/capacity-type=spot 6 3}]
I0110 13:05:45.195088       1 webhook.go:389] flow=CREATE serviceInstanceNum=67 nodeLabel=karpenter.sh/capacity-type=on-demand currently runs 0 pods
I0110 13:05:45.195107       1 webhook.go:393] flow=CREATE serviceInstanceNum=67 Currently running 0 pods is less than expected 4, scheduling pod on nodeLabel karpenter.sh/capacity-type=on-demand
I0110 13:05:45.195130       1 webhook.go:232] serviceInstanceNum=67 AdmissionResponse: patch=[{"op":"add","path":"/spec/nodeSelector","value":{"karpenter.sh/capacity-type":"on-demand"}}]
I0110 13:05:45.195150       1 webhook.go:311] Ready to write reponse ...
....
I0110 13:05:45.315759       1 webhook.go:378] flow=CREATE serviceInstanceNum=73 nodeLabelStrategyList=[{karpenter.sh/capacity-type=on-demand 4 1} {karpenter.sh/capacity-type=spot 6 3}]
I0110 13:05:45.339624       1 webhook.go:393] flow=CREATE serviceInstanceNum=73 Currently running 0 pods is less than expected 6, scheduling pod on nodeLabel karpenter.sh/capacity-type=spot
I0110 13:05:45.339643       1 webhook.go:232] serviceInstanceNum=73 AdmissionResponse: patch=[{"op":"add","path":"/spec/nodeSelector","value":{"karpenter.sh/capacity-type":"spot"}}]
I0110 13:05:45.339661       1 webhook.go:311] Ready to write reponse ...

./check_pod_spread.sh
NAME                     READY   STATUS    RESTARTS   AGE
nginx-6b6769fd96-fwwtk   1/1     Running   0          2m58s
.....
Number of Pods in namespace test is 1
Number of Pods for on-demand is 4
Number of Pods for spot is 6

kubectl scale deployment nginx  --replicas=20 -n test

jp:~/environment/custom-kubernetes-scheduler/admissionwebhook (main) $ ./check_pod_spread.shNAME                     READY   STATUS    RESTARTS   AGE
nginx-6b6769fd96-2qt9h   1/1     Running   0          19s
...
Number of Pods in namespace test is 20
Number of Pods for on-demand is 6
Number of Pods for spot is 14

kubectl delete deployment nginx -n test
kubectl delete ns test
kubectl delete -f $WEBHOOK_CONTROLLER
kubectl delete -f $WEBHOOK_CONFIG
kubectl delete secret $SECRET -n $NAMESPACE 
kubectl delete ns $NAMESPACE
# if you created a new EKS cluster, then delete the deleter
eksctl delete cluster  --name $CLUSTER_NAME  --region $AWS_REGION