FinOps 时代的高可用经济实惠 NAT 网关替代方案

AutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      MinSize: "2"
      MaxSize: "2"
      VPCZoneIdentifier: !Ref SubnetIDs
      MixedInstancesPolicy:
        InstancesDistribution:
          OnDemandBaseCapacity: 0
          OnDemandPercentageAboveBaseCapacity: 0
          SpotAllocationStrategy: "lowest-price"
        LaunchTemplate:
          LaunchTemplateSpecification:
            LaunchTemplateId: !Ref LaunchTemplate
            Version: !GetAtt LaunchTemplate.LatestVersionNumber
  LaunchTemplate:
    Type: "AWS::EC2::LaunchTemplate"
    Properties:
      LaunchTemplateName: "SpotInstanceLaunchTemplate"
      LaunchTemplateData:
        InstanceType: !Ref InstanceType
        ImageId: !Ref LatestAmiId 
        IamInstanceProfile:
          Name: !Ref InstanceProfile
        NetworkInterfaces:
          - DeviceIndex: 0
            AssociatePublicIpAddress: true
            DeleteOnTermination: true
            Groups:
              - !Ref SecurityGroup
        UserData:
          Fn::Base64: |

#cloud-config
              cloud_final_modules:
              - [scripts-user, always]

              runcmd:
                - |
                  #!/bin/bash

                  # Install awscurl and pip if not already installed

                  if ! command -v pip &> /dev/null; then
                    yum install -y pip
                  fi

                  if ! command -v awscurl &> /dev/null; then
                      pip install awscurl
                  fi

                  #BASH_ACTION="ModifyInstanceAttribute"
                  BASH_SOURCE_DEST_CHECK="false"
                  #VERSION="2016-11-15"
                  # Request a token for IMDSv2
                  tkn=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")

                  # Extract AWS temporary credentials from instance metadata using the token
                  crd=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/iam/security-credentials/)
                  ak=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/iam/security-credentials/${crd} | grep AccessKeyId | cut -d'"' -f4)
                  sk=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/iam/security-credentials/${crd} | grep SecretAccessKey | cut -d'"' -f4)
                  ss_tkn=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/iam/security-credentials/${crd} | /bin/sed -n 's/.*"Token" : "\(.*\)",/\1/p')
                  mac_id=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s  http://169.254.169.254/latest/meta-data/network/interfaces/macs)
                  eni_id=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/network/interfaces/macs/{$mac_id}interface-id)

                  #Get az information
                  az=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/placement/availability-zone/)
                  if [ $? -ne 0 ]
                  then
                      exit 255
                  fi

                  # Validate the az
                  if [[ ! "${az}" =~ ^([a-z]+-){2,3}[0-9][a-z]$ ]]
                  then
                      exit 255
                  fi

                  #Extract region from az
                  rgn=$(/bin/echo "${az}" | /bin/sed -n 's/\(\([a-z]\+-\)\+[0-9]\+\).*/\1/p')

                  # Get domain_name for calls
                  domain_name=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/services/domain/)
                  if [ $? -ne 0 ]
                  then
                      exit 255
                  fi

                  endpoint_name="ec2.${rgn}.${domain_name}"
                  ec2_pnt="https://${endpoint_name}"


                  # Get the instance ID and availability az
                  instance_id=$(curl -H "X-aws-ec2-metadata-token: $tkn" -s http://169.254.169.254/latest/meta-data/instance-id)

                  counter=0
                  while [ $counter -lt 10 ]; do
                      output=$(awscurl --service ec2 \
                        --region ${rgn}\
                        --access_key ${ak}\
                        --secret_key ${sk}\
                        --session_token ${ss_tkn}\
                      "${ec2_pnt}/?Action=ModifyInstanceAttribute&InstanceId=${instance_id}&SourceDestCheck.Value=${BASH_SOURCE_DEST_CHECK}&Version=2016-11-15")
                      
                      # 检查输出是否为成功消息
                      if [[ $output == *"<return>true</return>"* ]]; then
                          echo "Success!"
                          break
                      else
                          echo "Attempt $(($counter + 1)) failed. Retrying..."
                          counter=$((counter + 1))
                          sleep 2 # 等待2秒再试
                      fi
                  done

                  counter=0
                  while [ $counter -lt 10 ]; do
                      output=$(awscurl --service ec2 \
                        --region ${rgn}\
                        --access_key ${ak}\
                        --secret_key ${sk}\
                        --session_token ${ss_tkn}\
                        "${ec2_pnt}/?Action=CreateTags&ResourceId.1=${instance_id}&Tag.1.Key=Name&Tag.1.Value=nat(${eni_id: -4:4})-${az}&Version=2016-11-15")

                      # 检查输出是否为成功消息
                      if [[ $output == *"<return>true</return>"* ]]; then
                          echo "Success!"
                          break
                      else
                          echo "Attempt $(($counter + 1)) failed. Retrying..."
                          counter=$((counter + 1))
                          sleep 2 # 等待2秒再试
                      fi
                  done
                  # NAT configuration
                  sudo sysctl -w net.ipv4.ip_forward=1 | sudo tee -a /etc/sysctl.conf
                  sudo yum install -y nftables
                  sudo nft add table nat
                  sudo nft -- add chain nat prerouting { type nat hook prerouting priority -100 \; }
                  sudo nft add chain nat postrouting { type nat hook postrouting priority 100 \; }
                  sudo nft add rule nat postrouting oifname "$(ip -o link show device-number-0 | awk -F': ' '{print $2}')" masquerade
                  # NAT 设定保存
                  sudo nft list table nat | sudo tee /etc/nftables/al2023-nat.nft
                  echo 'include "/etc/nftables/al2023-nat.nft"' | sudo tee -a /etc/sysconfig/nftables.conf
                  # 启动+自动启动设定 
                  sudo systemctl start nftables
                  sudo systemctl enable nftables

Type: 'AWS::SSM::Parameter::Value<String>'
Default: '/aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-6.1-arm64'

def check_websites_reachable(urls, nat_instance_ids):
    urls = parse_input(urls)
    nat_instance_ids = parse_input(nat_instance_ids)

    route_tables = get_route_tables_by_tag('monitor')

    for nat_instance_id in nat_instance_ids:
        # Delete the 0.0.0.0/0 route only once at the beginning
        for route_table in route_tables:
            delete_route(route_table['RouteTableId'], "0.0.0.0/0")

        for route_table in route_tables:
            current_nat_id = False
            if not update_route_table(route_table['RouteTableId'], current_nat_id, nat_instance_id, '0.0.0.0/0'):
                logger.warning(
                    f"Route table {route_table['RouteTableId']} not updated correctly for NAT instance {nat_instance_id}, current_nat_id={current_nat_id}, nat_instance_id={nat_instance_id}.")
                return False

        logger.info(f"Using {nat_instance_id} to check")

        if check_websites_reachable_concurrently(urls):
            # Delete the 0.0.0.0/0 route only once after the check
            for route_table in route_tables:
                delete_route(route_table['RouteTableId'], "0.0.0.0/0")
            return True

    return False

def handle_route_updates(healthy_nat_instances, route_table, subnet_az, assigned_instances):
    rt_instance = None
    need_adjust = True 
    assigned_nat_instance = assigned_instances.get(route_table['RouteTableId'], None)
    nat_instances_in_az = [nat for nat in healthy_nat_instances if nat['AvailabilityZone'] == subnet_az]
    unassigned_in_az = [nat['NatInstanceId']for nat in nat_instances_in_az if nat['NatInstanceId'] not in assigned_instances.values()]
    other_az_instances = [nat for nat in healthy_nat_instances if nat['AvailabilityZone']!= subnet_az and nat['NatInstanceId'] not in assigned_instances.values()]
    
    # If the current instance is located in the same Availability Zone (AZ), there's no need for any modifications.
    for route in route_table['Routes']:
      if route.get('DestinationCidrBlock', False) == '0.0.0.0/0':
        rt_instance = route.get('InstanceId', False)
        if rt_instance and rt_instance in [nat['NatInstanceId'] for nat in healthy_nat_instances] and rt_instance in [nat['NatInstanceId'] for nat in nat_instances_in_az]:
            if rt_instance in unassigned_in_az:
              need_adjust = False
              assigned_instances[route_table['RouteTableId']]=rt_instance
            unassigned_in_az = [nat['NatInstanceId']
                    for nat in nat_instances_in_az if nat['NatInstanceId'] not in assigned_instances.values()]
            logger.info(f"rt_instance: {rt_instance} in route_table_id: {route_table['RouteTableId']} does not need adjustment") 
        
    if need_adjust:
      # Attempt to locate a NAT instance within the same AZ that hasn't been assigned yet.
      if unassigned_in_az:
        new_nat_id = random.choice(unassigned_in_az)
        assigned_instances[route_table['RouteTableId']] = new_nat_id
      else:
        # If there are no available unassigned instances, select a functioning instance from the same AZ.
        if nat_instances_in_az:
          new_nat_id = random.choice(nat_instances_in_az)['NatInstanceId']
          assigned_instances[route_table['RouteTableId']] = new_nat_id
        else:
            other_az_instances = [nat for nat in healthy_nat_instances if nat['AvailabilityZone']!= subnet_az and nat['NatInstanceId'] not in assigned_instances.values()]  
            #  If the current instance is located in a different Availability Zone, there's no need for any modifications.
            if rt_instance and rt_instance in other_az_instances:
                need_adjust = False
                assigned_instances[route_table['RouteTableId']]=rt_instance
            else:
                # If there's no instance available in the same AZ, choose one from a different AZ that hasn't been assigned yet.
                other_az_instances = [nat for nat in healthy_nat_instances if nat['AvailabilityZone']!= subnet_az and nat['NatInstanceId'] not in assigned_instances.values()]
                if other_az_instances:
                  new_nat_id = random.choice(other_az_instances)['NatInstanceId']
                  assigned_instances[route_table['RouteTableId']] = new_nat_id
                else:
                    # If there are no unassigned instances from other AZs, select a functioning instance.
                    if not other_az_instanc and healthy_nat_instances:
                        new_nat_id = random.choice(healthy_nat_instances)['NatInstanceId']
                                         
    if need_adjust and new_nat_id:
        update_route_table(route_table['RouteTableId'], rt_instance, new_nat_id, '0.0.0.0/0')

def is_port_open(ip, port):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(1)
    try:
        s.connect((ip, port))
        s.close()
        return True
    except socket.error:
        return False


def parse_input(input_data):
    return input_data.split(',') if isinstance(input_data, str) else input_data


def check_website_reachable(url, max_retries=3):
    parsed_url = urlparse(url)
    port = parsed_url.port if parsed_url.port else 443 if parsed_url.scheme == 'https' else 80

    retries = 0
    while retries < max_retries:
        if is_port_open(parsed_url.hostname, port):
            logger.info(f"{parsed_url.hostname}:{port} is reachable")
            return True
        retries += 1

    logger.warning(
        f"{parsed_url.hostname}:{port} is not reachable after {max_retries} retries")
    return False