Replace certificate for an isolated service running on TrueNAS SCALE and restart the service if needed

replace-cert.sh

#!/bin/bash
#set -e
set -o errexit -o pipefail -o noclobber -o nounset

if [ "$#" -ne 4 ]; then
    echo "Usage: ${0} CertTargetDir CertName OutputFormat<pem|p12> AssociatedService"
    exit 1
fi

certTargetDir="${1}"
certName="${2}"
outFormat="${3}"
assSvc="${4}"
certTargetBundle="${certTargetDir}/${certName}.${outFormat}"
certSrcCrt="/etc/certificates/${certName}.crt"
certSrcKey="/etc/certificates/${certName}.key"
assSvcStopTimeout=60
assSvcStartTimeout=600

# ===============================================================
# Gets textual service status
# Params: serviceId
# Prints: service status name (e.g. ACTIVE) or nothing on error
# Returns: 0 if command succedded, 1 otherwise
getSvcStatus () {
  _svcId="${1}"
  _svcStatus=$(midclt call chart.release.query "[[\"id\",\"=\",\"${_svcId}\"]]" | jq -r '.[0].status')
  if [[ "$_svcStatus" == "" ]] || [[ "${_svcStatus}" == "null" ]]; then
    return 1
  fi

  echo "${_svcStatus}"
}

# Loops until timeout is achieved or until the job completed
# Params: JobID Timeout
# Prints: logs
# Return: 0 if the job completed succesfully, 1 on job completion error or timeout
awaitJobCompletion () {
  _jobId="${1}"
  _timeout=$2

  for ((_timeLeft=_timeout; _timeLeft!=0; _timeLeft--)); do
    _jobStatus=$(midclt call core.get_jobs "[[\"id\",\"=\",$_jobId]]" | jq -r '.[0].state')
    case "${_jobStatus}" in
      null)
        echo "ERROR: Internal error - failed to query job \"$_jobId\" status"
        exit 1
        ;;

      WAITING | RUNNING)
        echo "Job#$_jobId is ${_jobStatus} - awaitng completion for ${_timeLeft} more seconds..."
        sleep 1
        ;;

      SUCCESS)
        echo "Job#$_jobId is ${_jobStatus} - OK"
        return 0
        ;;

      *)
        echo "ERROR: Job#$_jobId did not complete successfully (status=${_jobStatus})"
        return 1
        ;;
    esac
  done

  echo "ERROR: Job#\"$_jobId\" timeout!"
  return 1
}

# Installs PEM certificate from CRT and KEY files
# Params: <none>
# Prints: logs
# Return: <none>
installPEM () {
  # A small hack to ensure new lines (https://unix.stackexchange.com/a/583381)
  grep -h '' "${certSrcCrt}" "${certSrcKey}" > "${certTargetBundle}"
  echo "New PEM certificate installed at ${certTargetBundle}"
}

# Installs PKCS12 certificate from CRT and KEY files
# Params: <none>
# Prints: logs
# Return: <none>
installPKCS12 () {
  openssl pkcs12 -passout pass: -export \
	-in "${certSrcCrt}" \
	-inkey "${certSrcKey}" \
  	-out "${certTargetBundle}"
  echo "New PKCS12 certificate installed at ${certTargetBundle}"
}

# ===============================================================

# Probably misspelled target - don't try to create it to avoid confusion
if [[ ! -d "${certTargetDir}" ]]; then
    echo "ERROR: Certificate target directory \"${certTargetDir}\" doesn't exist"
    exit 1
fi

# Ensure certificates were already generated by ACME/CertBot
if [[ ! -f "${certSrcCrt}" ]]; then
    echo "ERROR: Source certificate file \"${certSrcCrt}\" doesn't exist"
    exit 1
fi
if [[ ! -f "${certSrcKey}" ]]; then
    echo "ERROR: Source private-key file \"${certSrcKey}\" doesn't exist"
    exit 1
fi

# Check if format is supported
if [[ "${outFormat}" != "pem" ]] && [[ "${outFormat}" != "p12" ]]; then
  echo "ERROR: Output format \"${outFormat}\" is not supported"
  exit 1
fi

# Ensure associated service exists and can be queried
if ! (getSvcStatus "${assSvc}" > /dev/null); then
  echo "ERROR: Associated service \"${assSvc}\" status unknown (does it exist?)"
  exit 1
fi

# ===============================================================

# If this is a first run the bundle may not exist - this can be safely fixed
if [[ -f "${certTargetBundle}" ]]; then
  newCertSN=$(openssl x509 -noout -serial -in "${certSrcCrt}" |  sed 's/.*=//g')
  if [[ "${outFormat}" == "pem" ]]; then
    oldCertSN=$(openssl x509 -noout -serial -in "${certTargetBundle}" |  sed 's/.*=//g')
  elif [[ "${outFormat}" == "p12" ]]; then
    oldCertSN=$(openssl pkcs12 -in "${certTargetBundle}" -clcerts -nokeys -password pass: | openssl x509 -noout -serial |  sed 's/.*=//g')
  fi
  if [[ "${newCertSN}" == "${oldCertSN}" ]]; then
    echo "Both new and current certificates are the same (S/N: ${newCertSN}) - nothing to do"
    exit 0
  fi
  echo "New certificate (S/N: ${newCertSN}) is different from the current one (S/N: ${oldCertSN}) - replacement needed"
else
    echo "WARN: Certificate target bundle \"${certTargetDir}\" doesn't exist (yet). It will be created now."
fi

# A small hack to ensure new lines (https://unix.stackexchange.com/a/583381)
if [[ "${outFormat}" == "pem" ]]; then
  installPEM
elif [[ "${outFormat}" == "p12" ]]; then
  installPKCS12
fi

# I think TNS has many architecture astronauts :D There's no simple way to just restart something. We need to carefully play with the API here.
# First, check if the service is running or currently being deployed - if it's not we shouldn't touch it as it was probably manually stopped.
# Available statuses: https://github.com/truenas/middleware/blob/2e95f545524ebeff91e58a5ff7273f22909841f4/src/middlewared/middlewared/plugins/chart_releases_linux/resources.py#L204
svcStatus=$(midclt call chart.release.query "[[\"id\",\"=\",\"${assSvc}\"]]" | jq -r '.[] | .status')
if [[ "${svcStatus}" != "ACTIVE" ]] && [[ "${svcStatus}" != "DEPLOYING" ]]; then
  echo "WARN: Service \"${assSvc}\" couldn't be restarted as it's not active (status=${svcStatus})"
  exit 0 #we assume it's successful as we replaced the certificate after all
fi

# "Stopping" the service is in fact scaling the application down to 0 replicas and can take some time
# The API will return job id which we need to monitor
echo "Attempting to stop \"$assSvc\"..."
svcStopJobId=$(midclt call chart.release.scale "${assSvc}" '{"replica_count":0}')
if ! (awaitJobCompletion "${svcStopJobId}" $assSvcStopTimeout); then
  echo "Service \"$assSvc\" failed to stop - aborting"
  exit 1
fi

echo "Attempting to start \"${assSvc}\"..."
svcStartJobId=$(midclt call chart.release.scale "${assSvc}" '{"replica_count":1}')
if ! (awaitJobCompletion "${svcStartJobId}" $assSvcStopTimeout); then
  echo "Service \"$assSvc\" failed to start - aborting"
  exit 1
fi

echo "SUCCESS: New certificate has been installed to \"${certTargetBundle}\" and the associated service \"${assSvc}\" has been restarted"