EMR 任务访问 S3 出现 credential 获取失败问题的分析与解决方案

org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException: No AWS Credentials provided by SimpleAWSCredentialsProvider EnvironmentVariableCredentialsProvider InstanceProfileCredentialsProvider : com.amazonaws.SdkClientException: Unable to load credentials from service endpoint
at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:187)
at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111)
at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:265)
…….
Caused by: com.amazonaws.SdkClientException: Unable to load credentials from service endpoint
at com.amazonaws.auth.EC2CredentialsFetcher.handleError(EC2CredentialsFetcher.java:183)
at com.amazonaws.auth.EC2CredentialsFetcher.fetchCredentials(EC2CredentialsFetcher.java:162)
at com.amazonaws.auth.EC2CredentialsFetcher.getCredentials(EC2CredentialsFetcher.java:82)
at com.amazonaws.auth.InstanceProfileCredentialsProvider.getCredentials(InstanceProfileCredentialsProvider.java:172)
at org.apache.hadoop.fs.s3a.AWSCredentialProviderList.getCredentials(AWSCredentialProviderList.java:137)

$ TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` && curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/MyInstanceRole
< HTTP/1.1 200 OK
< X-Aws-Ec2-Metadata-Token-Ttl-Seconds: 21600
< Content-Type: text/plain
< Accept-Ranges: none
< Last-Modified: Fri, 26 Jan 2024 00:57:20 GMT
< Content-Length: 1166
< Date: Fri, 26 Jan 2024 01:31:51 GMT  //发起请求时间
< Server: EC2ws
< Connection: close
<
{
  "Code" : "Success",
  "LastUpdated" : "2024-01-26T00:56:48Z",  //token 有效期开始时间
  "Type" : "AWS-HMAC",
  "AccessKeyId" : "ABCDEFTLI5CPBOFAH73",
  "SecretAccessKey" : "z1iQaGLLJuqIxgEyiQ9y06Om0nOtBqRrxGX4shy4",
  "Token" : "FwoDYXdzEBEaDAegS77iK14K+QcyvSKtBCYkUYHCWbRnsy4x9j2v3UqLjPCn1gcGhG+8dD7OKZtBg5rUUJsrBq9wHy8oPqDfQ7GVv/ntRrYp+YvoWgua3avsBG4pT5oTe8ubcSENzVUgq0p4TgguaXdpNv128QiTi14LR6eImTbi9z6ao+nud7l8L03G2ZVtIX1GbZWf+z+MRCWTEx7HBuUEoZQHYVRn3ZslXLtEc4nW6xSwV+IYfioZoDHtO5olY2UOkWmGi1dpioBBEr1gDf53Ll/M/h8np0wimAik0Bg4cwkaNniyAMS3r8npG3tb9Qp568+6UB5CQaAOP3gS/z4zGTqbwyCtAcBnPzwOIoUrclKhjiI5bzV8jG9Z/oBhYtUvXm+jlPJEfBC4TZ66weI6OvgwukPoh0O5E1leiJAUPM4AkaMCzU67xItV/UE7emiA/4Di4B7d1/IPmFlia2XQ+4NgHdJQ+gq62Fy9WVFBxZcwre2Q07HIOoNf3ov2H6vU8wJCrz7lUOR5Nw0rVZ2nC7ineWVbcoGxs7Q2UhnBVQ8iFqXhvPf518T/Nx2jkufp09UBHiquFKg+47jTsqEbCuD7U4PUuWo9WWzxtQJ/f2966VzK135FEhDVWRgKcWkG5lI9+Zsx5ph5opXdda6Rpa5qRUfagiRqn9ElTF9o37LWW5splpoQ28Xpwz2rMIxz3vWa8zMqOnqMve9CbzYiSbwYHTZ0DcdYhfEnmXffSMPjKUwZruz578rLQxX+LiHmj1iCKPCGzK0GMl3cJ72X6I5EHbs2KFCQUAeW2SnlNXJKV8RbAytcgno9lnS692Zc1SEg5EIxS47BanmCLfOqbk25eozE3HDPGpZ7+oCIbX0d2Qzjzuu5XMj19ByhnfU854K7mk2GLMc=",
  "Expiration" : "2024-01-26T07:18:16Z"  //token 有效期结束时间

com.amazonaws.auth.AWSCredentialsProvider
com.amazonaws.auth.InstanceProfileCredentialsProvider
InstanceProfileCredentialsProvider
  private synchronized void fetchCredentials() {
    if (!needsToLoadCredentials())
      return; 
    try {
      this.lastInstanceProfileCheck = new Date();
      String credentialsResponse = EC2CredentialsUtils.getInstance().readResource(this.credentialsEndpointProvider
          .getCredentialsEndpoint(), this.credentialsEndpointProvider
          .getRetryPolicy(), this.credentialsEndpointProvider
          .getHeaders());
      JsonNode node = (JsonNode)Jackson.fromSensitiveJsonString(credentialsResponse, JsonNode.class);
      JsonNode accessKey = node.get("AccessKeyId");
      JsonNode secretKey = node.get("SecretAccessKey");
      JsonNode token = node.get("Token");
      if (null == accessKey || null == secretKey)
        throw new SdkClientException("Unable to load credentials."); 
      if (null != token) {
        this
          .credentials = new BasicSessionCredentials(accessKey.asText(), secretKey.asText(), token.asText());
      } else {
        this
          .credentials = new BasicAWSCredentials(accessKey.asText(), secretKey.asText());
      } 
      JsonNode expirationJsonNode = node.get("Expiration");
      if (null != expirationJsonNode) {
        String expiration = expirationJsonNode.asText();
        expiration = expiration.replaceAll("\\+0000$", "Z");
        try {
          this.credentialsExpiration = DateUtils.parseISO8601Date(expiration);
        } catch (Exception ex) {
          handleError("Unable to parse credentials expiration date from Amazon EC2 instance", ex);
        } 
      } 
    } catch (JsonMappingException e) {
      handleError("Unable to parse response returned from service endpoint", (Exception)e);
    } catch (IOException e) {
      handleError("Unable to load credentials from service endpoint", e);
    } catch (URISyntaxException e) {
      handleError("Unable to load credentials from service endpoint", e);
    } 
  }

//刷新本地 token 的条件
  protected boolean needsToLoadCredentials() {
    if (this.credentials == null)  //如果本地没有 credential
      return true; 
    if (this.credentialsExpiration != null && isWithinExpirationThreshold()) //本地的 credential 已过期或者是还有 15 分钟到期
      return true; 
    if (this.lastInstanceProfileCheck != null && isPastRefreshThreshold()) //上次刷新在一小时前
      return true; 
    return false;
  }

 private boolean isWithinExpirationThreshold() {
    return (this.credentialsExpiration.getTime() - System.currentTimeMillis() < 900000L);
  }

private boolean isPastRefreshThreshold() {
    return (System.currentTimeMillis() - this.lastInstanceProfileCheck.getTime() > 3600000L);
  }
  
  private boolean expired() {
    if (this.credentialsExpiration != null && 
      this.credentialsExpiration.getTime() < System.currentTimeMillis())
      return true; 
    return false;
  }

public AWSCredentials getCredentials() {
    checkNotEmpty();
    if (this.reuseLastProvider && this.lastProvider != null)
      return this.lastProvider.getCredentials(); 
    AmazonClientException lastException = null;
    for (AWSCredentialsProvider provider : this.providers) {
      try {
        AWSCredentials credentials = provider.getCredentials();
        if ((credentials.getAWSAccessKeyId() != null && credentials
          .getAWSSecretKey() != null) || credentials instanceof com.amazonaws.auth.AnonymousAWSCredentials) {
          this.lastProvider = provider;
          LOG.debug("Using credentials from {}", provider);
          return credentials;
        } 
      } catch (AmazonClientException e) {
        lastException = e;
        LOG.debug("No credentials provided by {}: {}", new Object[] { provider, e
              .toString(), e });
      } 
    } 
    String message = "No AWS Credentials provided by " + listProviderNames();
    if (lastException != null)
      message = message + ": " + lastException; 
    throw new AmazonClientException(message, lastException);
  }

uptime
 15:16:11 up 236 days, 19:15,  0 users,  load average: 195.19, 157.28, 137.45
 14:35:14 up 318 days, 23:12,  0 users,  load average: 117.48, 137.96, 103.15

public AWSCredentials getCredentials() {
        if (needsToLoadCredentials()){
         try{  
              fetchCredentials(); 
              }catch(Exception e){
                    if ( this.credentials !=null){
                          return this.credentials
                       }else{
                         Throw exception e;
                       }
               }
          }
}

@override
 public AWSCredentials getCredentials() {
        if (needsToLoadCredentials()){
         try{  
              fetchCredentials(); 
              }catch(Exception e){
                    if ( this.credentials !=null){
                          return this.credentials
                       }else{
                         Throw exception e;
                       }
               }
          }
}