Terraform external data source example - dynamic SSH key generation

ssh_key.tf

# ssh key generator data source expects the below 3 inputs, and produces 3 outputs for use:
#  "${data.external.ssh_key_generator.result.public_key}" (contents)
#  "${data.external.ssh_key_generator.result.private_key}" (contents)
#  "${data.external.ssh_key_generator.result.private_key_file}" (path)
data "external" "ssh_key_generator" {
  program = ["bash", "${path.root}/../ssh_key_generator.sh"]

  query = {
    customer_name = "${var.customer_name}"
    customer_group = "${var.customer_group}"
    customer_environment = "${var.customer_environment}"
  }
}

resource "aws_key_pair" "admin" {
  key_name   = "${var.customer_name}-${var.customer_group}-${var.customer_environment}"
  public_key = "${data.external.ssh_key_generator.result.public_key}"
}

ssh_key_generator.sh

#!/bin/bash
# ssh_key_generator - designed to work with the Terraform External Data Source provider
#   https://www.terraform.io/docs/providers/external/data_source.html
#  by Irving Popovetsky <irving@popovetsky.com> 
#
#  this script takes the 3 customer_* arguments as JSON formatted stdin
#  produces public_key & private_key (contents) and the private_key_file (path) as JSON formatted stdout
#  DEBUG statements may be safely uncommented as they output to stderr

function error_exit() {
  echo "$1" 1>&2
  exit 1
}

function check_deps() {
  test -f $(which ssh-keygen) || error_exit "ssh-keygen command not detected in path, please install it"
  test -f $(which jq) || error_exit "jq command not detected in path, please install it"
}

function parse_input() {
  # jq reads from stdin so we don't have to set up any inputs, but let's validate the outputs
  eval "$(jq -r '@sh "export CUSTOMER_NAME=\(.customer_name) CUSTOMER_GROUP=\(.customer_group) CUSTOMER_ENVIRONMENT=\(.customer_environment)"')"
  if [[ -z "${CUSTOMER_NAME}" ]]; then export CUSTOMER_NAME=none; fi
  if [[ -z "${CUSTOMER_GROUP}" ]]; then export CUSTOMER_GROUP=none; fi
  if [[ -z "${CUSTOMER_ENVIRONMENT}" ]]; then export CUSTOMER_ENVIRONMENT=none; fi
}

function create_ssh_key() {
  script_dir=$(dirname $0)
  export ssh_key_file="${script_dir}/.ssh/${CUSTOMER_NAME}-${CUSTOMER_GROUP}-${CUSTOMER_ENVIRONMENT}"
  # echo "DEBUG: ssh_key_file = ${ssh_key_file}" 1>&2
  if [[ ! -f "${ssh_key_file}" ]]; then
    ssh-keygen -q -t rsa -N '' -f $ssh_key_file
  fi
}

function produce_output() {
  public_key_contents=$(cat ${ssh_key_file}.pub)
  # echo "DEBUG: public_key_contents ${public_key_contents}" 1>&2
  private_key_contents=$(cat ${ssh_key_file} | awk '$1=$1' ORS='  \n')
  # echo "DEBUG: private_key_contents ${private_key_contents}" 1>&2
  # echo "DEBUG: private_key_file ${ssh_key_file}" 1>&2
  jq -n \
    --arg public_key "$public_key_contents" \
    --arg private_key "$private_key_contents" \
    --arg private_key_file "$ssh_key_file" \
    '{"public_key":$public_key,"private_key":$private_key,"private_key_file":$private_key_file}'
}

# main()
check_deps
# echo "DEBUG: received: $INPUT" 1>&2
parse_input
create_ssh_key
produce_output

ssh_key_generator_example_output.txt

$ echo '{"customer_name": "foo", "customer_group": "bbar", "customer_environment": "baz"}' | ./ssh_key_generator.sh
{
  "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgeAzqoZZnHg04V4zbI21FwzL2Aw0fhAYZlblyQsZhqYLSiVhge9zTP63x1hbN2f0L5ZmtXw1eeBgNJJHK91UJyDaF7+J8llNvizYeiFqWLyJiotXgvNZIe2ms9eeWyEer3g2W74YlnGKL+5UiM+1dw44Es3vRV6A8M4oZJLUKZxSl6Kzo128ua71Fv6HuOiTfInThFMtPjeTlOIaXt7tq0nzkAmxnMU2+EtCPKq01MmlR0bC5GxnSwrFMwSD8FwOU0jiJ7t+HT4BRRaYvp36/6HkiMBVqSnFlz21cJiKKRzlH7Ssl+R5wVsShXmX5+Sp24vP+uyWmKhxxVC7/wqqP irving@computer.local",
  "private_key": "-----BEGIN RSA PRIVATE KEY-----  \nMIIEogIBAAKCAQEA4HgM6qGWZx4NOFeM2yNtRcMy9gMNH4QGGZW5ckLGYamC0olY  \nYHvc0z+t8dYWzdn9C+WZrV8NXngYDSSRyvdVCcg2he/ifJZTb4s2Hohali8iYqLV  \n4LzWSHtprPXnlshHq94Nlu+GJZxii/uVIjPtXcOOBLN70VegPDOKGSS1CmcUpeis  \n6NdvLmu9Rb+h7jok3yJ04RTLT43k5TiGl7e7atJ85AJsZzFNvhLQjyqtNTJpUdGw  \nuRsZ0sKxTMEg/BcDlNI4ie7fh0+AUUWmL6d+v+h5IjAVakpxZc9tXCYiikc5R+0r  \nJfkecFbEoV5l+fkqduLz/rslpioccVQu/8KqjwIDAQABAoIBAHabWI/d5AAGpAui  \nTz43gPS8yL+vKw79DtAUChIy8GoITKT8h6Mrr6o72qiPbCtHROs1Xbd7IzBImsTP  \nDu5FNDzf+tdYwr78G4gz8du+RsdWjn+59PM0NLHF7DfFE6LbnutUgK/BTouvD29R  \n9yJEd+b0fqVDRWh/OZ61yQGyIKsmgL3X5iOkDUN3GTkOuT1XMBNU3RYMyQGQcUU0  \n+y4WUpFrjuKtCBdS8RNHqPvKxsfifnr4tlFmWtQHbCjGbdDur0vbpq9IP8J3piAD  \nm8xv9Wj+ulsSMzHCsn4QN2YKP+sAjibg/W34TVcHr6pfK690oHV4+Q5hzMHvOZLd  \n3ACPP9ECgYEA/myNZluC+q3RxcG9msvnCva34zOzqmTnF24Q9EXNAp7dzU2FMK2l  \nh1ZBq0e06KISb5PZkOxfjVTVqYoCzrSdseKp0wuaubVj3l9WBdPnrqY5qkjgbzWx  \ne/hqbKOjF+c/HxOXqbUt/NMO7RBW9U6cR2aExpo4CdHwMW4rJUETZOcCgYEA4dv/  \nWSA0o0leR+q8jpTqNhPRqAxNTRPkT4biX57mc5Ln4w+l3XGX1j0rtgQ1/L05WCwc  \nYg1D9MygTPY6mEcKmuqFDUjuneafzUS6cAwO7h1klzLRNBP7YAnY6KI0FqA4DMiX  \nrq0vngXVrBjqHtXl2ALdn2ZAH3X0kFdBgFWpsBkCgYBgjYOPz7TCO0q7mM3CvBTf  \nRUf90jYhuQ82BhArE348u1uDOSMNmSiTVrmvLZRLII6Mh3huljWg5gv7viNYnJSn  \n2FQIgoPibCMNVfLIXWW0EuMZa3S435COcnS469TOEnUS7xWEUvyz0Mj+UFAf4ghO  \n1GoZEJepqmFT8PIwviSFCwKBgCokJiy2+ZtN4S2B+tSPrHOSlxfH09SB1aORA0Pc  \nHhuKWYHgNY5v12i92R4JAxm5JK3y7QjOeNOAKpixiJVJDA2DnHeyF/OWSFLAdBjb  \n5x0+lrovXSFeaRSuQa6GNTnTgyG/e6232p6dcBTAQU6nkk8PmdJX/bbhB1S3Mx2C  \n3jphAoGAScrlJ6R287CDekndxNsJRwC345ePJARZRxNgxN/8Xww2wauPGhXcVJ0E  \ntZtH8/mCrm3xO2VqWAlKugmXzHO1TihhoEaY42P4XwxSighjKaDjHtbMpWAqJI0e  \niB2QaFQfli2YxMbV7gxGV3/sL8mgsjMognYun8K7GNMj1jBTTic=  \n-----END RSA PRIVATE KEY-----  ",
  "private_key_file": "./.ssh/foo-bbar-baz"
}

Thanks for this writeup @irvingpop. Used this to implement a workaround for a AWS Batch Terraform bug here

Thanks a lot @irvingpop!

I've modified input parsing to this bloc to not have to define all query keys:

input=$(jq -r '.')
# echo "DEBUG: input: ${input}" 1>&2
for query_value in $(echo "${input}" | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]'); do
  export ${query_value}
done

Beware, I noticed that currently Terraform don't handle well query value composed of JSON.
My workaround is to encode it with base64 on Terraform side and decode it on bash side.

Thank you for sharing the script.
For those who will be reading this, pay attention at the additional result key in the line:

${data.external.ssh_key_generator.result.public_key}

One more simplified example is here: https://stackoverflow.com/a/55188305/1929406

Is the result data stored in the state file, or just in memory?

+1 @kivagant-ba

For those who will be reading this, pay attention at the additional result key in the line:

${data.external.ssh_key_generator.result.public_key}

This should be in the documentation at

• https://www.terraform.io/docs/providers/external/data_source.html

hi, any idea how to return a list from bash to terraform? for cidr_blocks in https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group

hi, any idea how to return a list from bash to terraform? for cidr_blocks in https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group

there's an open feature request:
hashicorp/terraform-provider-external#2

Invalid length for parameter DBProxyName, value: 0, valid min length: 1 ( i getting above said error while implementing) any clue on this?

hey all, please don't use this anymore except as an example for external data sources. the version of terraform this was written for has long been deprecated. Use the tls_private_key resource instead, in the tls provider: https://registry.terraform.io/providers/hashicorp/tls/latest/docs

hey all, please don't use this anymore. the version of terraform this was written for has long been deprecated. Use the tls_private_key resource instead, in the tls provider: https://registry.terraform.io/providers/hashicorp/tls/latest/docs

It's still an excellent example of how to use external data source and call a program. Sure, the application here has been superseded by built-in functionality in later versions of terraform, but the approach demonstrated is flawless.

🥰 Thank you @scooper4711 that's very kind of you, I updated my comment

+1 for this being a master-class in how to properly leverage really useful external data sources.

@seboudry

Thanks a lot @irvingpop!

I've modified input parsing to this bloc to not have to define all query keys:

input=$(jq -r '.')
# echo "DEBUG: input: ${input}" 1>&2
for query_value in $(echo "${input}" | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]'); do
  export ${query_value}
done

Beware, I noticed that currently Terraform don't handle well query value composed of JSON. My workaround is to encode it with base64 on Terraform side and decode it on bash side.

I have modified this to show the base64encoding since I ran into the same problem when using jsonencode(value) and pushed the value into an array:

#!/bin/bash

# Handle stdin from Terraform "external" data source
# A parameter "query" of type map(string) is passed to stdin
# In order to control the expected output, parameters can use base64 encoding
# ex: query = {
#       "mount_points" = base64encode(jsonencode(var.machine.spec.additional_volumes[*].mount_point))
#       "ssh_user"     = base64encode(var.operating_system.ssh_user)
#       "ip_address"   = base64encode(aws_instance.machine.public_ip)
#     }
# Expected: {
#  "ip_address": "NTIuOTEuMjMwLjEzNQ==",
#  "mount_points": "WyIvb3B0L3BnX2RhdGEiLCIvb3B0L3BnX3dhbCJd",
#  "ssh_user": "cm9ja3k="
# }
# Grab stdin with 'jq' and
# insert decoded values into an associative array
TERRAFORM_INPUT=$(jq '.')
declare -A INPUT_MAPPING
for key in $(echo "${TERRAFORM_INPUT}" | jq -r 'keys_unsorted|.[]'); do
    INPUT_MAPPING["$key"]=$(echo "$TERRAFORM_INPUT" | jq -r .[\"$key\"] | base64 -d)
    # INPUT_MAPPING["ip_address"]=52.91.164.228
    # INPUT_MAPPING["mount_points"]=["/opt/pg_data","/opt/pg_wal"]
    # INPUT_MAPPING["ssh_user"]=rocky
    echo "DEBUG: key value: $key ${INPUT_MAPPING["$key"]}" >> /tmp/terraform.log
done

# stdout must be returned as a json object
# object saved to 'result': data.external.<name>.result
# stderr passed through to terraform as is
jq -n --arg arg0 "$TERRAFORM_INPUT" '{"passed":$arg0}'