AWS FSx deployment scripts

destroy_fsx.sh

#!/bin/bash

#
# Destroy FSx deployment
#
# Reference: https://docs.aws.amazon.com/fsx/latest/ONTAPGuide/what-is-fsx-ontap.html

set -eo pipefail

# Check if DEBUG_MODE is set and enable bash -x
if [[ ! -z ${DEBUG_MODE:-} ]]; then
  set -x
fi

readonly SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
readonly TOP_DIR=$(cd "${SCRIPT_DIR}"; git rev-parse --show-toplevel)

[[ -z ${CLUSTER_DIR} ]] && export CLUSTER_DIR=/tmp

source "${SCRIPT_DIR}/func.sh"

# Delete all volumes attached to the SVM
echo " ⚙ Delete all volumes attached to the SVM"
getSvmId | xargs -I{} aws fsx describe-volumes --filters Name="storage-virtual-machine-id",Values="{}" --no-paginate \
  | jq '.Volumes[] | select(.OntapConfiguration.JunctionPath != "/") | .VolumeId' \
  | xargs -I{} aws fsx delete-volume --client-request-token {} --volume-id {} --ontap-configuration SkipFinalBackup=true --no-paginate | jq '.Lifecycle'

echo " 🔥 All volumes attached to the SVM are deleted"


# Delete the SVM
echo " ⚙ Delete the SVM"
getSvmId | xargs -I{} aws fsx delete-storage-virtual-machine --storage-virtual-machine-id {} --no-paginate || true

# Do until the FSx deployment is deleted
while : ; do
    deploymentStatus=$(getSvmStatus)

    if [[ ${deploymentStatus} == 'DELETING' ]]; then
        echo " ⏳ Wait for SVM deployment to be deleted"
        sleep 30
        continue
    fi

    if [[ ${deploymentStatus} == 'StorageVirtualMachineNotFound' ]] ; then
      echo " 🔥 SVM deleted"
      break
    fi
done

# Delete the FSx
echo " ⚙ Delete the FSx"
destroyFSx && sleep 5

# Do until the FSx deployment is deleted
while : ; do
    deploymentStatus=$(getFileSystemStatus)

    if [[ ${deploymentStatus} == 'DELETING' ]]; then
        echo " ⏳ Wait for FSx deployment to deleted"
        sleep 30
        continue
    fi

    [[ ${deploymentStatus} == 'FileSystemNotFound' ]] && break

done

echo " 🔥 FSx deleted"

func.sh

#!/usr/bin/env bash

[[ -z ${CLUSTER_DIR} ]] && export CLUSTER_DIR=/tmp
logfile="${CLUSTER_DIR}/fsx.json"
svm_state_file="${CLUSTER_DIR}/svm.json"
vol_state_file="${CLUSTER_DIR}/fsx.volume.json"
fsx_name=cnv-qe-${CLUSTER_NAME:-'ci'}
svm_name=svm1
vol_name=vol1

getSubnetByVPC() {
  local vpc_id=$1
  aws ec2 describe-subnets \
    | jq --arg vpc_id "${vpc_id}" -r '.Subnets[] | select(.VpcId == "${vpc_id}") | .SubnetId'
}

getSubnetsId() {
#   aws ec2 describe-subnets | jq -r '.Subnets[] | .SubnetId' | tail -1 | tr '\n' ' ' | sed 's/ *$//'
  aws ec2 describe-subnets | jq -r '.Subnets[] | .SubnetId' | xargs echo
}

abortIfStateFileNotFound() {
  local state_file=${1}
  if [[ ! -f "${state_file}" ]]; then
    echo " 🛑 Can't find log file [${state_file}]"
    exit 1
  fi
}

getFileSystemId() {
  abortIfStateFileNotFound "${logfile}"
  jq -r .FileSystem.FileSystemId "${logfile}"
}

getFileSystemStatus() {
  # abortStateogFileNotFound
  local file_system_not_found_json='{"FileSystems":[{"Lifecycle":"FileSystemNotFound"}]}'
  getFileSystemId \
    | (xargs -I{} aws fsx describe-file-systems --no-paginate --file-system-id {} || echo "${file_system_not_found_json}") \
    | jq -r '.FileSystems[0].Lifecycle'
}

deployFSx() {
  aws fsx create-file-system \
    --file-system-type=ONTAP \
    --storage-type=SSD \
    --client-request-token="${client_request_token}" \
    --storage-capacity=${storage_capacity} \
    --tags ${tags} \
    --ontap-configuration="${ONTAP_CONFIGURATION}" \
    --subnet-ids=$(getSubnetsId) \
    --region=${region} \
    ${debug} | tee "${logfile}"
}
# TODO:
# Before deleteing the FSx deployment,
#  one need to first remove the volumes,
#  and the Storage virtual machines (SVMs)
#
destroyFSx() {
  abortIfStateFileNotFound "${logfile}"
  jq .FileSystem.FileSystemId "${logfile}" | xargs -I{} aws fsx delete-file-system --file-system-id {} --no-paginate | jq -r .Lifecycle
}

createSVM() {
  aws fsx create-storage-virtual-machine \
    --client-request-token "${client_request_token}" \
    --file-system-id "$(getFileSystemId)" \
    --name "${svm_name}" \
    --svm-admin-password "${SvmAdminPassword}" \
    --root-volume-security-style UNIX \
    --no-cli-pager \
    --tags ${tags} | tee "${CLUSTER_DIR}/svm.json"
}

getSvmId() {
  abortIfStateFileNotFound "${svm_state_file}"
  jq -r .StorageVirtualMachine.StorageVirtualMachineId "${svm_state_file}"
  # aws fsx describe-storage-virtual-machines | jq -r '.StorageVirtualMachines[0].StorageVirtualMachineId'
}

getSvmStatus() {
  local storage_virtual_machine_not_found='{"StorageVirtualMachines":[{"Lifecycle":"StorageVirtualMachineNotFound"}]}'
  getSvmId \
    | (xargs -I{} aws fsx describe-storage-virtual-machines --storage-virtual-machine-ids {} || echo "${storage_virtual_machine_not_found}") \
    | jq -r '.StorageVirtualMachines[0].Lifecycle'
}

createVolume() {
  aws fsx create-volume \
    --no-cli-pager \
    --client-request-token "${client_request_token}" \
    --volume-type "ONTAP" \
    --name "${vol_name}" \
    --ontap-configuration "${VOLUME_CREATE_CONFIG}" \
    | tee "${vol_state_file}"
}

getVolumeId() {
  abortIfStateFileNotFound "${vol_state_file}"
  jq -r .Volume.VolumeId "${vol_state_file}"
}

getVolumeStatus() {
  getVolumeId | xargs -I{} aws fsx describe-volumes --volume-ids {} \
  | jq -r '.Volumes[0].Lifecycle'
}

setup_fsx.sh

#!/bin/bash

#
# Setup FSx.
#
# Reference: https://docs.aws.amazon.com/fsx/latest/ONTAPGuide/what-is-fsx-ontap.html

set -euo pipefail

# Check if DEBUG_MODE is set and enable bash -x
if [[ ! -z ${DEBUG_MODE:-} ]]; then
  set -x
fi

readonly SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
readonly TOP_DIR=$(cd "${SCRIPT_DIR}"; git rev-parse --show-toplevel)

source "${SCRIPT_DIR}/func.sh"

# Minimum storage capacity for ONTAP file systems is 1024
storage_capacity=1024
region=us-east-1
# subnet_ids=$(getSubnetsId)
tags="Key=Name,Value=${fsx_name} Key=${CLUSTER_TAG:-MyClusterTag},Value=owned"
client_request_token=${fsx_name}
debug="--debug"
debug=""
SvmAdminPassword='super-secret-password!2'

# # Get information about a specific subnet
# aws ec2 describe-subnets --subnet-ids subnet-id --region your-region

# # Get information about subnets in a specific VPC
# aws ec2 describe-subnets --filters "Name=vpc-id,Values=vpc-id" --region your-region

### TODO: if MultiZone
# - PreferredSubnetId
# - EndpointIpAddressRange
# - RouteTableIds
# - DiskIopsConfiguration
#    Iops

ONTAP_CONFIGURATION=$(jq --monochrome-output --raw-output --compact-output << __JSON__
  {
    "AutomaticBackupRetentionDays": 0,
    "DailyAutomaticBackupStartTime": "00:00",
    "DeploymentType": "SINGLE_AZ_1",
    "FsxAdminPassword": "super-secret-password!1",
    "DiskIopsConfiguration": {
      "Mode": "AUTOMATIC"
    },
    "ThroughputCapacity": 1024,
    "WeeklyMaintenanceStartTime": "6:00:00"
  }
__JSON__
)

# Create FSx file system
echo " 🚀 Deploy FSx file system (~30 minutes)"
deployFSx && sleep 5

while : ; do
  deploymentStatus=$(getFileSystemStatus)

  if [[ ${deploymentStatus} == 'CREATING' ]]; then
    echo " ⏳ Wait for FSx deployment to finish [status=${deploymentStatus}]"
    sleep 30
    continue
  fi

  if [[ ${deploymentStatus} == 'AVAILABLE' ]]; then
    echo " 🎉 FSx deployment finished"
    break
  fi

done

# Create SVM inside the FSx file system
echo " 🚀 Deploy SVM"
createSVM

while : ; do
  deploymentStatus=$(getSvmStatus)

  if [[ ${deploymentStatus} == 'CREATING' ]]; then
    echo " ⏳ Wait for SVM deployment to finish [status=${deploymentStatus}]"
    sleep 30
    continue
  fi

  if [[ ${deploymentStatus} == 'CREATED' ]]; then
    echo " 🎉 SVM deployment finished"
    break
  fi

done

# Privileged-delete must be permanently disabled for SnapLock Compliance volume.

VOLUME_CREATE_CONFIG=$(jq --monochrome-output --raw-output --compact-output << __JSON__
{
  "JunctionPath": "/vol1",
  "SecurityStyle": "UNIX",
  "SizeInMegabytes": 777,
  "StorageEfficiencyEnabled": true,
  "StorageVirtualMachineId": "$(getSvmId)",
  "TieringPolicy": {
    "Name": "SNAPSHOT_ONLY"
  },
  "OntapVolumeType": "RW",
  "SnapshotPolicy": "none",
  "CopyTagsToBackups": true,
  "SnaplockConfiguration": {
    "SnaplockType": "COMPLIANCE",
    "AuditLogVolume": false,
    "AutocommitPeriod": {
      "Type": "NONE"
    },
    "PrivilegedDelete": "PERMANENTLY_DISABLED",
    "RetentionPeriod": {
      "DefaultRetention": {
        "Type": "UNSPECIFIED"
      },
      "MinimumRetention": {
        "Type": "DAYS",
        "Value": 0
      },
      "MaximumRetention": {
        "Type": "DAYS",
        "Value": 7
      }
    }
  }
}
__JSON__
)

# Create volume inside the SVM
echo " ✨ Create volume"
createVolume

while : ; do
  deploymentStatus=$(getVolumeStatus)

  if [[ ${deploymentStatus} == 'CREATING' ]]; then
    echo "Wait for Volume creation to finish [status=${deploymentStatus}]"
    sleep 30
    continue
  fi

  if [[ ${deploymentStatus} == 'CREATED' ]]; then
    echo " 🎉 Volume creation finished"
    break
  fi

done