Hadoop S3A prepare creds from another account

temporary_creds.sh

#!/bin/sh

# more info:
# https://hadoop.apache.org/docs/r3.0.3/hadoop-aws/tools/hadoop-aws/s3guard.html
# https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html
# disable S3Guard 
# hdfs dfs -Dfs.s3a.s3a-bucket.metadatastore.impl=org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore -ls s3a://s3a-bucket/
# Trust Relationship for CrossAccountRole in account 222222 allows access from 123456:
#. 
#   {
#   "Version": "2012-10-17",
#   "Statement": [
#     {
#       "Effect": "Allow",
#       "Principal": {
#         "AWS": [
#           "arn:aws:iam::123456:role/generic-server"
#         ]
#       },
#       "Action": "sts:AssumeRole",
#       "Condition": {}
#     }
#   ]
# }

# external aws account  
ACCOUNT_ID=123456
creds=$(aws sts assume-role --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CrossAccountRole --role-session-name  CrossAccountRole)


tmpfile=$(mktemp /tmp/sts.XXXXXX)
rm -f ${tmpfile}
hadoop credential create fs.s3a.access.key -value $(echo ${creds} | jq -r .Credentials.AccessKeyId) \
    -provider localjceks://file${tmpfile}

hadoop credential create fs.s3a.secret.key -value $(echo ${creds} | jq -r .Credentials.SecretAccessKey) \
    -provider localjceks://file${tmpfile}
# it's possible to store the creds on HDFS with the following 
# -provider jceks://hdfs/user/hadoop/s3.jceks
### 
# store on hdfs:
# grep -o 'hdfs://.*8020' /etc/hadoop/conf/core-site.xml
# -provider jceks://hdfs@$(grep -o 'hdfs://.*8020' /etc/hadoop/conf/core-site.xml)
# Note:
# you need right Hadoop user
# export HADOOP_USER_NAME=hadoop

if [[ $(hadoop credential list -provider localjceks://file${tmpfile} |grep  fs.s3a|wc -l) -eq 2 ]];then
    ln -sf /etc/hadoop/conf /etc/hadoop/conf.ext
    cat ${tmpfile}  > /etc/hadoop/conf.ext/s3a.jceks
    echo "hdfs dfs -Dhadoop.security.credential.provider.path=jceks://file/etc/hadoop/conf.ext/s3a.jceks -ls s3a://s3a-bucket/"

fi