[setup-local-s3-with-minio] setup local s3 with minio #minio #setp #s3 #s3a

setup-local-s3-with-minio.sh

#!/bin/bash
set -x
#set -eu -o pipefail

# file: CI/start-minio.sh

export minio_data_dir='/opt/deptmnt/data/minio-data'
mkdir -p $minio_data_dir

# set credentials for the server
export MINIO_ACCESS_KEY=N9Z4W68XW1WM7P9MF6WG
export MINIO_SECRET_KEY=R2g5sRfT1xou2xfaMoq+pQrLLIB8+erOVRhtNg2F
export MINIO_REGION="us-east-1"

echo MINIO_ACCESS_KEY: $MINIO_ACCESS_KEY
echo MINIO_SECRET_KEY: $MINIO_SECRET_KEY
echo MINIO_REGION: $MINIO_REGION

test $(pgrep minio|wc -l) -gt 0 && kill -9 $(pgrep minio) # kill if running

/usr/local/bin/minio server "$minio_data_dir" 2>&1 &  # and start the server

export CONFIG_S3_BUCKET_NAME='deptmnt-local-bucket'
export CONFIG_S3_ENDPOINT='https://127.0.0.1:9000'
export CONFIG_S3_ACCESS_KEY=$MINIO_ACCESS_KEY
export CONFIG_S3_SECRET_KEY=$MINIO_SECRET_KEY
export CONFIG_S3_REGION=$MINIO_REGION
export CONFIG_S3_DISABLE_SSL_CHECK='true'

echo CONFIG_S3_BUCKET_NAME: $CONFIG_S3_BUCKET_NAME
echo CONFIG_S3_ENDPOINT: $CONFIG_S3_ENDPOINT
echo CONFIG_S3_ACCESS_KEY: $CONFIG_S3_ACCESS_KEY
echo CONFIG_S3_SECRET_KEY: $CONFIG_S3_SECRET_KEY
echo CONFIG_S3_REGION: $CONFIG_S3_REGION
echo CONFIG_S3_DISABLE_SSL_CHECK: $CONFIG_S3_DISABLE_SSL_CHECK

export MINIO_HOST_ALIAS='minio_host_alias'
# register the host alias
/usr/local/bin/mc config host add $MINIO_HOST_ALIAS "$CONFIG_S3_ENDPOINT" "$CONFIG_S3_ACCESS_KEY" \
   "$CONFIG_S3_SECRET_KEY" --api S3v4

# create the bucket
/usr/local/bin/mc --insecure mb --region "$MINIO_REGION" "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"

# configure the policies on the bucket
/usr/local/bin/mc --insecure policy upload "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"
/usr/local/bin/mc --insecure policy download "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"
/usr/local/bin/mc --insecure policy public "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"
/usr/local/bin/mc --insecure policy list "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"

/usr/local/bin/mc --insecure policy upload "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing"
/usr/local/bin/mc --insecure policy download "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing"
/usr/local/bin/mc --insecure policy public "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing"
/usr/local/bin/mc --insecure policy list "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing"

# ls the buckets in the host alias
/usr/local/bin/mc --insecure ls $MINIO_HOST_ALIAS

/usr/local/bin/mc --insecure cp /etc/livy/livy.conf \
$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing/livy.conf

# and verify it is there
echo ls $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME
/usr/local/bin/mc --insecure ls $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME
echo stat $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing/livy.conf
/usr/local/bin/mc --insecure stat $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing/livy.conf
/usr/local/bin/mc --insecure policy download "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing/livy.conf"
echo ls $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing/

/usr/local/bin/mc --insecure ls $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME/landing/

export AWS_ACCESS_KEY=$MINIO_ACCESS_KEY
export AWS_SECRET_KEY=$MINIO_SECRET_KEY

mkdir -p ~/.aws/s3cmd/ ; cat << EOF03 > ~/.aws/s3cmd/minio_local.s3cfg
  host_base = 127.0.0.1:9000
  host_bucket = ava-local-bucket
  bucket_location = us-east-1
  use_https = True
  access_key =  N9Z4W68XW1WM7P9MF6WG
  secret_key = R2g5sRfT1xou2xfaMoq+pQrLLIB8+erOVRhtNg2F
  signature_v2 = False
EOF03


HTTPS_PROXY=""
HTTP_PROXY=""
ALL_PROXY=""
https_proxy=""
http_proxy=""
#aws --debug --no-verify-ssl --endpoint-url 'https://127.0.0.1:9000' s3 ls "s3://$CONFIG_S3_BUCKET_NAME/"
s3cmd ls --no-check-certificate -r -c ~/.aws/s3cmd/minio_local.s3cfg "s3://$CONFIG_S3_BUCKET_NAM