Terraform external data source to generate SOPS-encrypted secret files

README.md

Terraform gen-sops

This example demonstrates how to use terraform's external data provider to automatically encrypt and save sops secrets files.

This implementation makes every effort to avoid exposing secrets in stdout, or by writing unencrypted data to temporary files on disk.

The example includes:

1. gen-sops.sh script (generates and returns sops-encrypted file contents to terraform)
2. test_gen-sops.sh script which provides an easy way to continuously test your script
3. terraform example implementation (vars.tf, sops_secrets.tf)

Best-practices:

• marks sensitive data (vars) as sensitive
• avoids exposing secrets in stdout or log output (in tf + in the shell script)
• avoids writing unencrypted data to any temporary filesystem locations
• writes the sops file to disk using the local_sensitive_file terraform resource

gen-sops.sh

#!/bin/bash
#
# sopsfile generator - designed to work with the Terraform External Data Source provider
#   https://www.terraform.io/docs/providers/external/data_source.html
#  by Armen Rostamian (StelthLabs) <armen@stelth.io>
#
#  This script takes the 3 arguments as JSON-formatted stdin
#  It produces the file content for a sops-encrypted secrets file as JSON-formatted stdout
#  The JSON-formatted stdout is then written to disk as YAML by the terraform module)...
#  because YAML is easier on the human brain, and just as friendly to the machine
#
#  DEBUG statements may be safely uncommented as they only output to stderr

function error_exit() {
  echo "$1" 1>&2
  exit 1
}

function check_deps() {
  test -f $(which sops) || error_exit "sops not detected in path, please install it -> https://github.com/mozilla/sops"
  test -f $(which jq) || error_exit "jq not detected in path, please install it -> https://github.com/stedolan/jq"
  test -f $(which jo) || error_exit "jo not detected in path, please install it -> https://github.com/jpmens/jo"
}

function parse_input() {
  # jq reads from stdin so we don't have to set up any inputs, but let's validate the outputs
  eval "$(jq -r '@sh "export KMS_KEY_ARN=\(.kms_key_arn) ENCRYPTION_PROFILE=\(.encryption_profile) export SENSITIVE_MATERIAL=\(.sensitive_material)"')"

  if [[ -z "${KMS_KEY_ARN}" ]]; then export KMS_KEY_ARN=none; fi
  if [[ -z "${ENCRYPTION_PROFILE}" ]]; then export ENCRYPTION_PROFILE=none; fi

  if [[ -z "${SENSITIVE_MATERIAL}" ]]; then
    export SENSITIVE_MATERIAL=none
  else
    DECODED_MATERIAL=$(echo "${SENSITIVE_MATERIAL}" | base64 -d)
    export SENSITIVE_MATERIAL="${DECODED_MATERIAL}"
  fi

  # DEBUG EXAMPLE
  # echo "ENCRYPTION_PROFILE: $ENCRYPTION_PROFILE" 1>&2
}

function render_data_output() {
  # !! READ BEFORE CHANGING !!
  # 
  # Changing directories ensures that sops doesn't get confused
  # because its default behaviour is to traverse the filesystem (upward)
  # until it finds a .sops.yaml config file.
  # To short-circuit this behaviour we change directories to a location
  # like "/tmp" where we don't keep or expect to find any sops configs.

  cd /tmp/ || exit

  ENCRYPTED_DATA=$(echo "$SENSITIVE_MATERIAL" \
    | sops \
        --encrypt \
        --kms $KMS_KEY_ARN \
        --aws-profile "$AWS_PROFILE" \
        --input-type json \
        --output-type json \
      /dev/stdin \
    | base64)

  jo encrypted_content="$ENCRYPTED_DATA"
}

# Do useful work, friend...
check_deps
parse_input
render_data_output

sops_secrets.tf

locals {
  # We take a map of sensitive secrets and:
  # 1. jsonencode it
  # 2. base64 the json-encoded object
  # 3. conveniently pass that to sopsfile_generator
  #
  # ...this makes the secrets data easier to pass around and use
  
  stringified_json_encoded_secrets_content = base64encode(jsonencode(var.generated_secrets))

  # The gen-sops script retruns a base64-encoded JSON string with just 1 key/value pair.
  # 
  # NOTE: the returned data has the following shape
  # ==>   {encrypted_content: "base64 encoded output from sops (the entire secrets-file content)"}
  #
  # 1. base64decode it
  # 2. jsondecode it
  # 3. yamlencode the resulting map
  # 4. write the resulting YAML to a file
  
  yaml_encoded_secrets_content = yamlencode(jsondecode(base64decode(data.external.sopsfile_generator.result.encrypted_content)))
  
}

data "external" "sopsfile_generator" {
  # !! Change this path to point to wherever the "gen-sops.sh" script
  #    resides, relative to your terraform module!
  program = ["bash", "${path.module}/../../lib/scripts/gen-sops.sh"]

  query = {
    kms_key_arn        = "${var.sops_key_alias_arn}"
    encryption_profile = "${var.sops_encryption_profile}"
    sensitive_material = "${local.stringified_json_encoded_secrets_content}"
  }
}

resource "local_sensitive_file" "generated_secrets_file" {
  content  = local.yaml_encoded_secrets_content
  filename = var.secrets_file_destination
}

test_gen-sops.sh

#!/bin/bash

# !! Change the following values (or export correct values before running this script)
KEY_ARN=arn:aws:kms:us-west-2:<ACCOUNT_ID>:alias/<KEY_ALIAS>
ENCRYPTION_PROFILE=<my_configured_AWS_cli_profile>

# You can leave this as-is...it's just nonsensical data
SENSITIVE_MATERIAL="eyJhc3BlcmEiOnsiQVNQRVJBX1BBU1MiOiJTb21lRGVwZW5kZW5jeU91dFB1dHN0dWZzYXNkZmtsYWphc2RmaXdlcmlwdWFza2ZqYXNkZiIsIkFTUEVSQV9VU0VSIjoiU29tZURlcGVuZGVuY3lPdXRQdXRzdHVmc2FzZGZrbGFqYXNkZml3ZXJpcHVhc2tmamFzZGYifSwiZHJtIjp7IkRSTV9DRVJUIjoiT3RoZXJUeXBlc09mRGF0YVRoYXRTaG91bGQxMjNCZUVuY3J5cHRlZCEhISIsIkRSTV9DRVJUX0ZJTEUiOiJPdGhlclR5cGVzT2ZEYXRhVGhhdFNob3VsZDEyM0JlRW5jcnlwdGVkISEhIiwiRFJNX0tFWSI6Ik90aGVyVHlwZXNPZkRhdGFUaGF0U2hvdWxkMTIzQmVFbmNyeXB0ZWQhISEifSwibGl2ZXJvb21zIjp7IkxJVkVST09NU19BQ0NPVU5UX0lEIjoiT3RoZXJUeXBlc09mRGF0YVRoYXRTaG91bGQxMjNCZUVuY3J5cHRlZCEhISIsIkxJVkVST09NU19BUElfU0VDUkVUIjoiT3RoZXJUeXBlc09mRGF0YVRoYXRTaG91bGQxMjNCZUVuY3J5cHRlZCEhISJ9LCJtZWRpYSI6eyJNRURJQV9TM19BQ0NFU1NfS0VZIjoiU29tZURlcGVuZGVuY3lPdXRQdXRzdHVmc2FzZGZrbGFqYXNkZml3ZXJpcHVhc2tmamFzZGYiLCJNRURJQV9TM19TRUNSRVRfS0VZIjoiU29tZURlcGVuZGVuY3lPdXRQdXRzdHVmc2FzZGZrbGFqYXNkZml3ZXJpcHVhc2tmamFzZGYifSwicGVuZG8iOnsiUEVORE9fQVBJX0tFWSI6bnVsbH0sInNsYWNrIjp7IlNMQUNLX0hPT0tfRU5EUE9JTlQiOiJPdGhlclR5cGVzT2ZEYXRhVGhhdFNob3VsZDEyM0JlRW5jcnlwdGVkISEhIn0sIndvd3phIjp7IldPV1pBX1BBU1MiOiJTb21lRGVwZW5kZW5jeU91dFB1dHN0dWZzYXNkZmtsYWphc2RmaXdlcmlwdWFza2ZqYXNkZiIsIldPV1pBX1VTRVIiOiJTb21lRGVwZW5kZW5jeU91dFB1dHN0dWZzYXNkZmtsYWphc2RmaXdlcmlwdWFza2ZqYXNkZiJ9fQ=="

# Test the script
jo kms_key_arn=$KEY_ARN encryption_profile=$ENCRYPTION_PROFILE sensitive_material=$SENSITIVE_MATERIAL \
    | ./generate_sopsfile.sh \
    | jq -r .encrypted_content \
    | base64 -d

# The json-encoded terraform map ("$SENSITIVE_MATERIAL") of test data looks like this:
#
# 
# {
#   "aspera": {
#     "ASPERA_PASS": "SomeDependencyOutPutstufsasdfklajasdfiweripuaskfjasdf",
#     "ASPERA_USER": "SomeDependencyOutPutstufsasdfklajasdfiweripuaskfjasdf"
#   },
#   "drm": {
#     "DRM_CERT": "OtherTypesOfDataThatShould123BeEncrypted!!!",
#     "DRM_CERT_FILE": "OtherTypesOfDataThatShould123BeEncrypted!!!",
#     "DRM_KEY": "OtherTypesOfDataThatShould123BeEncrypted!!!"
#   },
#   "liverooms": {
#     "LIVEROOMS_ACCOUNT_ID": "OtherTypesOfDataThatShould123BeEncrypted!!!",
#     "LIVEROOMS_API_SECRET": "OtherTypesOfDataThatShould123BeEncrypted!!!"
#   },
#   "media": {
#     "MEDIA_S3_ACCESS_KEY": "SomeDependencyOutPutstufsasdfklajasdfiweripuaskfjasdf",
#     "MEDIA_S3_SECRET_KEY": "SomeDependencyOutPutstufsasdfklajasdfiweripuaskfjasdf"
#   },
#   "pendo": {
#     "PENDO_API_KEY": null
#   },
#   "slack": {
#     "SLACK_HOOK_ENDPOINT": "OtherTypesOfDataThatShould123BeEncrypted!!!"
#   },
#   "wowza": {
#     "WOWZA_PASS": "SomeDependencyOutPutstufsasdfklajasdfiweripuaskfjasdf",
#     "WOWZA_USER": "SomeDependencyOutPutstufsasdfklajasdfiweripuaskfjasdf"
#   }
# }

vars.tf

variable "generated_secrets" {
  type        = any
  sensitive   = true
  description = "A Map of secrets you want to encrypt with sops (can be nested data)"
  default = null
}

variable "sops_key_alias_arn" {
  type        = string
  description = "The ARN of the KMS key alias used to encrypt the SOPS secrets"
  default     = null
}

variable "sops_encryption_profile" {
  type        = string
  description = "The name of the aws-cli profile sops will use to call KMS"
  default     = null
}

variable "secrets_file_destination" {
  type        = string
  description = "The path to the file where the SOPS secrets will be written"
  default     = null
}