基于 MIG 技术在 Amazon SageMaker HyperPod 上实现 GPU 虚拟化的最佳实践

# 添加NVIDIA官方Helm仓库
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

# 生产级GPU Operator安装
helm install gpu-operator nvidia/gpu-operator \
  --namespace gpu-operator \
  --create-namespace \
  --set deviceplugin.enabled=true \
  --set migmanager.enabled=true \
  --set migmanager.with_reboot=true \
  --set daemonsets.nodeSelector."eks\.amazonAmazon\.com/nodegroup"="gpu-nodegroup" \
  --set toolkit.version=v1.17.8-ubuntu20.04 \
  --set defaultRuntime=containerd \
  --set gfd.version=v0.17.3 \
  --set deviceplugin.version=v0.17.3 \
  --set migmanager.default=all-balanced \
  --set operator.defaultRuntime=containerd \
  --set driver.version="580.65.06" \
  --set dcgmExporter.enabled=true \
  --set dcgmExporter.serviceMonitor.enabled=true

# gpu-operator-values.yaml 企业级配置示例
daemonsets:
  nodeSelector:
    eks.amazonAmazon.com/nodegroup: "gpu-nodegroup"
  tolerations:
  - key: nvidia.com/gpu
    operator: Exists
    effect: NoSchedule
  - key: hyperpod.amazonAmazon.com/gpu
    operator: Exists
    effect: NoSchedule
  priorityClassName: system-node-critical
  updateStrategy: "RollingUpdate"
  rollingUpdate:
    maxUnavailable: "1"

migManager:
  enabled: true
  config:
    create: true
    name: enterprise-mig-config
    default: "all-balanced"
    data:
      config.yaml: |-
        version: v1
        mig-configs:
          all-disabled:
            - devices: all
              mig-enabled: false
          production-inference:
            - devices: all
              mig-enabled: true
              mig-devices:
                "3g.20gb": 2
                "1g.5gb": 2
          development-mixed:
            - devices: all
              mig-enabled: true
              mig-devices:
                "1g.5gb": 4
                "2g.10gb": 1

export NODE_NAME=hyperpod-i-01251a3ea13e02745

# 场景一：高并发轻量推理（适合API服务）
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-1g.5gb --overwrite
# 配置结果：7个实例，每个5GB显存，支持BERT、小型CNN等模型

# 场景二：中等规模推理（适合企业级应用）
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-2g.10gb --overwrite
# 配置结果：3个实例，每个10GB显存，支持GPT-3.5、中型视觉模型

# 场景三：大模型推理（适合ChatGPT类应用）
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-3g.20gb --overwrite
# 配置结果：2个实例，每个20GB显存，支持LLaMA-7B、GPT-4等大模型

# 场景四：混合策略（推荐生产环境）
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-balanced --overwrite
# 配置结果：2个1g.5gb + 1个2g.10gb + 1个3g.20gb，灵活适配不同负载

# 场景一：超高并发轻量推理
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-1g.18gb --overwrite
# 配置结果：7个实例，每个18GB显存，支持更大的轻量级模型

# 场景二：中等规模高性能推理
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-1g.35gb --overwrite
# 配置结果：4个实例，每个35GB显存，支持中型多模态模型

# 场景三：大模型推理的黄金配置
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-3g.71gb --overwrite
# 配置结果：2个实例，每个71GB显存，完美适配70B参数模型

# 场景四：超大模型的完整资源
kubectl label nodes $NODE_NAME nvidia.com/mig.config=all-7g.141gb --overwrite
# 配置结果：1个实例，完整141GB显存，支持175B+超大模型

# 步骤1：等待节点就绪
kubectl wait --for=condition=Ready node $NODE_NAME --timeout=600s

# 步骤2：验证MIG资源注册
kubectl describe node $NODE_NAME | grep nvidia.com/mig
# 预期输出：应该看到相应的MIG资源计数

# 步骤3：检查GPU Operator组件状态
kubectl get pods -n gpu-operator -o wide
# 确保所有Pod都处于Running状态

# 步骤4：验证MIG Manager日志
kubectl logs -n gpu-operator -l app=nvidia-mig-manager --tail=50
# 查找"MIG configuration applied successfully"消息

# 步骤5：检查设备插件状态
kubectl logs -n gpu-operator -l app=nvidia-device-plugin-daemonset --tail=50
# 确认MIG设备被正确识别和注册

# 部署综合测试套件
kubectl apply -f examples/quick-test.yaml

# 等待Pod启动
kubectl wait --for=condition=Ready pod -l app=test-h200-1g18gb -n mig-test --timeout=300s

# 验证MIG实例可见性
kubectl exec -n mig-test deployment/test-h200-1g-18gb -- nvidia-smi -L | grep MIG

# 执行性能基准测试
kubectl exec -n mig-test deployment/test-h200-1g-18gb -- nvidia-smi --query-gpu=name,memory.total,memory.used,utilization.gpu --format=csv

# 实时监控GPU使用情况
kubectl exec -it -n mig-test deployment/test-h200-1g-18gb -- nvidia-smi dmon -s pucvmet -d 1

# 创建详细的备份策略
BACKUP_DIR="backup/mig-migration-$(date +%Y%m%d_%H%M%S)"
mkdir -p $BACKUP_DIR

# 备份所有GPU相关的工作负载配置
kubectl get deployments,statefulsets,daemonsets --all-namespaces \
  -o json | jq '.items[] | select(.spec.template.spec.containers[].resources.requests["nvidia.com/gpu"])' \
  > $BACKUP_DIR/gpu-workloads.json

# 备份节点标签和注解
kubectl get nodes -o json | jq '.items[] | select(.metadata.labels["nvidia.com/gpu.present"]=="true")' \
  > $BACKUP_DIR/gpu-nodes.json

# 记录当前资源使用情况
kubectl top nodes --selector=node.kubernetes.io/instance-type=p4d.24xlarge \
  > $BACKUP_DIR/resource-usage-baseline.txt

# 生成迁移计划报告
echo "=== GPU节点迁移计划 ===" > $BACKUP_DIR/migration-plan.txt
kubectl get nodes -l node.kubernetes.io/instance-type=p4d.24xlarge \
  -o custom-columns=NAME:.metadata.name,INSTANCE:.metadata.labels.node\.kubernetes\.io/instance-type,ZONE:.metadata.labels.topology\.kubernetes\.io/zone \
  >> $BACKUP_DIR/migration-plan.txt

#!/bin/bash
# 智能滚动升级脚本

set -e

# 配置参数
MIGRATION_LOG="migration-$(date +%Y%m%d_%H%M%S).log"
MAX_PARALLEL_NODES=2  # 同时升级的最大节点数
HEALTH_CHECK_INTERVAL=30  # 健康检查间隔（秒）
ROLLBACK_THRESHOLD=3  # 失败重试次数

# 日志函数
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a $MIGRATION_LOG
}

# 健康检查函数
health_check() {
    local node=$1
    local max_attempts=10
    local attempt=1

    while [ $attempt -le $max_attempts ]; do
        if kubectl get node $node -o jsonpath='{.metadata.labels.nvidia\.com/mig\.config\.state}' | grep -q "success"; then
            log "节点 $node 健康检查通过"
            return 0
        fi
        log "节点 $node 健康检查失败，重试 $attempt/$max_attempts"
        sleep $HEALTH_CHECK_INTERVAL
        ((attempt++))
    done

    log "节点 $node 健康检查最终失败"
    return 1
}

# 获取所有GPU节点并分组
GPU_NODES=($(kubectl get nodes -l node.kubernetes.io/instance-type=p4d.24xlarge \
  --no-headers -o custom-columns=NAME:.metadata.name))

log "开始MIG迁移，总计 ${#GPU_NODES[@]} 个节点"

# 分批处理节点
for ((i=0; i<${#GPU_NODES[@]}; i+=MAX_PARALLEL_NODES)); do
    batch_nodes=("${GPU_NODES[@]:i:MAX_PARALLEL_NODES}")
    log "开始处理第 $((i/MAX_PARALLEL_NODES + 1)) 批节点: ${batch_nodes[*]}"

    # 并行处理当前批次的节点
    for node in "${batch_nodes[@]}"; do
        (
            log "开始升级节点: $node"

            # 1. 标记节点不可调度
            kubectl cordon $node
            log "节点 $node 已标记为不可调度"

            # 2. 优雅驱逐Pod
            kubectl drain $node \
                --ignore-daemonsets \
                --delete-emptydir-data \
                --grace-period=300 \
                --timeout=600s \
                --force
            log "节点 $node Pod驱逐完成"

            # 3. 应用MIG配置
            kubectl label nodes $node nvidia.com/mig.config=all-3g.20gb --overwrite
            log "节点 $node MIG配置已应用"

            # 4. 等待配置完成
            if health_check $node; then
                # 5. 恢复节点调度
                kubectl uncordon $node
                log "节点 $node 升级成功，已恢复调度"
            else
                log "节点 $node 升级失败，需要人工干预"
                exit 1
            fi
        ) &
    done

    # 等待当前批次完成
    wait
    log "第 $((i/MAX_PARALLEL_NODES + 1)) 批节点升级完成"

    # 批次间等待，确保集群稳定
    sleep 120
done

log "所有节点MIG迁移完成"
迁移过程的实时监控
# 部署迁移监控Dashboard
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: migration-monitor
  namespace: gpu-operator
data:
  monitor.sh: |
    #!/bin/bash
    while true; do
        echo "=== MIG迁移状态监控 $(date) ==="
        echo "GPU节点总数: $(kubectl get nodes -l node.kubernetes.io/instance-type=p4d.24xlarge --no-headers | wc -l)"
        echo "MIG已启用节点: $(kubectl get nodes -l nvidia.com/mig.config.state=success --no-headers | wc -l)"
        echo "配置中节点: $(kubectl get nodes -l nvidia.com/mig.config.state=pending --no-headers | wc -l)"
        echo "失败节点: $(kubectl get nodes -l nvidia.com/mig.config.state=failed --no-headers | wc -l)"
        echo ""
        kubectl get nodes -l node.kubernetes.io/instance-type=p4d.24xlarge \
          -o custom-columns=NAME:.metadata.name,MIG-CONFIG:.metadata.labels.nvidia\.com/mig\.config,STATE:.metadata.labels.nvidia\.com/mig\.config\.state
        echo "================================"
        sleep 60
    done
EOF

#!/bin/bash
# 应用分析和分类脚本

# 创建应用分析报告
ANALYSIS_DIR="app-analysis-$(date +%Y%m%d_%H%M%S)"
mkdir -p $ANALYSIS_DIR

# 发现所有GPU应用
echo "=== GPU应用发现与分析 ===" > $ANALYSIS_DIR/gpu-apps-analysis.txt

kubectl get deployments,statefulsets --all-namespaces -o json | \
  jq -r '.items[] | select(.spec.template.spec.containers[].resources.requests["nvidia.com/gpu"]) |
  {
    namespace: .metadata.namespace,
    name: .metadata.name,
    kind: .kind,
    replicas: .spec.replicas,
    gpu_request: .spec.template.spec.containers[0].resources.requests["nvidia.com/gpu"],
    cpu_request: .spec.template.spec.containers[0].resources.requests.cpu,
    memory_request: .spec.template.spec.containers[0].resources.requests.memory,
    image: .spec.template.spec.containers[0].image
  }' > $ANALYSIS_DIR/gpu-apps-inventory.json

# 根据资源需求进行应用分类
python3 << 'EOF'
import json
import sys

# 读取应用清单
with open('$ANALYSIS_DIR/gpu-apps-inventory.json', 'r') as f:
    apps = [json.loads(line) for line in f]

# 分类规则
categories = {
    'lightweight': [],  # 轻量级推理
    'medium': [],       # 中等规模推理
    'heavy': [],        # 大模型推理
    'training': []      # 训练任务
}

for app in apps:
    gpu_count = int(app.get('gpu_request', 1))
    memory = app.get('memory_request', '0Gi')
    memory_gb = float(memory.replace('Gi', '').replace('G', '')) if memory != '0Gi' else 0

    # 分类逻辑
    if 'training' in app['name'].lower() or 'train' in app['name'].lower():
        categories['training'].append(app)
    elif memory_gb <= 8:
        categories['lightweight'].append(app)
    elif memory_gb <= 32:
        categories['medium'].append(app)
    else:
        categories['heavy'].append(app)

# 生成迁移建议
with open('$ANALYSIS_DIR/migration-recommendations.json', 'w') as f:
    json.dump(categories, f, indent=2)

# 打印分类统计
for category, apps in categories.items():
    print(f"{category}: {len(apps)} applications")
EOF

#!/bin/bash
# 渐进式应用迁移脚本

# 迁移配置映射
declare -A MIG_MAPPING=(
    ["lightweight"]="nvidia.com/mig-1g.5gb"
    ["medium"]="nvidia.com/mig-2g.10gb"
    ["heavy"]="nvidia.com/mig-3g.20gb"
    ["training"]="nvidia.com/mig-7g.40gb"
)

# 迁移函数
migrate_application() {
    local namespace=$1
    local name=$2
    local kind=$3
    local mig_resource=$4

    echo "开始迁移应用: $namespace/$name ($kind)"

    # 创建迁移前快照
    kubectl get $kind $name -n $namespace -o yaml > "$ANALYSIS_DIR/${namespace}-${name}-backup.yaml"

    # 计算合适的CPU和内存配置
    local cpu_request="2"
    local cpu_limit="4"
    local memory_request="8Gi"
    local memory_limit="16Gi"

    case $mig_resource in
        "nvidia.com/mig-1g.5gb")
            cpu_request="1"; cpu_limit="2"
            memory_request="4Gi"; memory_limit="8Gi"
            ;;
        "nvidia.com/mig-2g.10gb")
            cpu_request="2"; cpu_limit="4"
            memory_request="8Gi"; memory_limit="16Gi"
            ;;
        "nvidia.com/mig-3g.20gb")
            cpu_request="4"; cpu_limit="8"
            memory_request="16Gi"; memory_limit="32Gi"
            ;;
        "nvidia.com/mig-7g.40gb")
            cpu_request="8"; cpu_limit="16"
            memory_request="32Gi"; memory_limit="64Gi"
            ;;
    esac

    # 应用MIG配置
    kubectl patch $kind $name -n $namespace --type='json' -p="[
        {
            \"op\": \"replace\",
            \"path\": \"/spec/template/spec/containers/0/resources/requests\",
            \"value\": {
                \"$mig_resource\": 1,
                \"cpu\": \"$cpu_request\",
                \"memory\": \"$memory_request\"
            }
        },
        {
            \"op\": \"replace\",
            \"path\": \"/spec/template/spec/containers/0/resources/limits\",
            \"value\": {
                \"$mig_resource\": 1,
                \"cpu\": \"$cpu_limit\",
                \"memory\": \"$memory_limit\"
            }
        }
    ]"

    # 等待滚动更新完成
    if kubectl rollout status $kind/$name -n $namespace --timeout=600s; then
        echo "应用 $namespace/$name 迁移成功"

        # 验证应用健康状态
        sleep 30
        if kubectl get pods -n $namespace -l app=$name -o jsonpath='{.items[*].status.phase}' | grep -q "Running"; then
            echo "应用 $namespace/$name 健康检查通过"
        else
            echo "警告: 应用 $namespace/$name 可能存在问题"
        fi
    else
        echo "错误: 应用 $namespace/$name 迁移失败"
        # 自动回滚
        kubectl apply -f "$ANALYSIS_DIR/${namespace}-${name}-backup.yaml"
        return 1
    fi
}

# 按优先级顺序迁移应用
for category in lightweight medium heavy training; do
    echo "开始迁移 $category 类别的应用..."

    # 从分类结果中读取应用列表
    apps=$(jq -r ".${category}[] | \"\(.namespace) \(.name) \(.kind)\"" $ANALYSIS_DIR/migration-recommendations.json)

    while IFS= read -r app_info; do
        if [ -n "$app_info" ]; then
            read namespace name kind <<< "$app_info"
            migrate_application "$namespace" "$name" "$kind" "${MIG_MAPPING[$category]}"

            # 应用间等待，避免集群压力过大
            sleep 60
        fi
    done <<< "$apps"

    echo "$category 类别应用迁移完成"
    sleep 120  # 类别间等待
done

# 适用于BERT、小型CNN等模型
apiVersion: apps/v1
kind: Deployment
metadata:
  name: lightweight-inference
spec:
  template:
    spec:
      containers:
      - name: inference
        resources:
          requests:
            nvidia.com/mig-1g.5gb: 1
            cpu: 1
            memory: 4Gi
          limits:
            nvidia.com/mig-1g.5gb: 1
            cpu: 2
            memory: 8Gi
        env:
        - name: CUDA_VISIBLE_DEVICES
          value: "all"
        - name: NVIDIA_VISIBLE_DEVICES
          value: "all"
      nodeSelector:
        nvidia.com/mig.capable: "true"
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule

# 适用于LLaMA-70B、GPT-4等大模型
apiVersion: apps/v1
kind: Deployment
metadata:
  name: large-model-inference
spec:
  template:
    spec:
      containers:
      - name: inference
        resources:
          requests:
            nvidia.com/mig-3g.71gb: 1
            cpu: 8
            memory: 32Gi
          limits:
            nvidia.com/mig-3g.71gb: 1
            cpu: 16
            memory: 64Gi
        env:
        - name: CUDA_VISIBLE_DEVICES
          value: "all"
        - name: NVIDIA_VISIBLE_DEVICES
          value: "all"
        # 大模型优化参数
        - name: CUDA_LAUNCH_BLOCKING
          value: "1"
        - name: NCCL_DEBUG
          value: "INFO"
      nodeSelector:
        nvidia.com/mig.capable: "true"
        node.kubernetes.io/instance-type: "p5.48xlarge"  # 指定H200节点
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule