SaltStack Module and Renderer for HashiCorp Vault

vault_module.py

# -*- coding: utf-8 -*-
'''
Execution module to work with HashiCorp's Vault

:depends:  - python-requests

In order to use an this module, a profile must be created in the master
configuration file:

Token example:

.. code-block:: yaml

    hashicorp_vault_config:
      url: https://127.0.0.1:8200
      verify: True
      keyname: saltstack
      token: 674r0488-9993-8545-140c-93af3c494at9

Client TLS Cert example:

.. code-block:: yaml

    hashicorp_vault_config:
      url: https://127.0.0.1:8200
      verify: True
      keyname: saltstack
      cert_pem: path/to/cert.pem
      key_pem: path/to/key.pem

Now that the configuration is done, Generate a encryption key. 
On your Vault sever, run:

.. code-block:: bash

    # vault write transit/keys/saltstack salt=stack

Now, to encrypt secrets, query the Vault server with a base64 encoding of your
string to receive a cipher text response:

.. code-block:: bash

    $ echo -n "the quick brown fox" | base64 | vault write transit/encrypt/saltstack plaintext=-

Set up the renderer on your master by adding something like this line to your
config:

.. code-block:: yaml

    renderer: jinja | yaml | vault

Now you can include your ciphers in your pillar data like so:

.. code-block:: yaml

    a-secret: vault:v0:czEwyKqGZY/limnuzDCUUe5AK0tbBObWqeZgFqxCuIqq7A84SeiOq3sKD0Y/KUvv
'''
from __future__ import absolute_import

# Import python libs
import json
import base64
import logging

# Import salt libs
from salt.exceptions import CommandExecutionError

# Import 3rd-party libs
import salt.ext.six as six
# pylint: disable=import-error
try:
    import requests
    HAS_LIBS = True
except ImportError:
    HAS_LIBS = False
# pylint: enable=import-error

__virtualname__ = 'vault'

log = logging.getLogger(__name__)

__func_alias__ = {
    'get_': 'get',
}


def __virtual__():
    '''
    Only return if python-requests is installed
    '''
    return __virtualname__ if HAS_LIBS else False


def get_(keyname, ciphertext, profile=None):
    '''
    .. versionadded:: 2015.7.0

    Get a value from vault transit backend, by key

    CLI Examples:

    .. code-block:: bash

        salt myminion vault.get keyname vault:v0:czEwyKqGZY/limnuzDCUUe5AK0tbBObWqeZgFqxCuIqq7A84SeiOq3sKD0Y/KUvv
    '''
    client = get_conn(__opts__, profile)
    return client.transit_decrypt(keyname, ciphertext)


def get_conn(opts, profile=None):
    '''
    .. versionadded:: 2014.15.0

    Return a client object for accessing vault
    '''
    opts_pillar = opts.get('pillar', {})
    opts_master = opts_pillar.get('master', {})

    opts_merged = {}
    opts_merged.update(opts_master)
    opts_merged.update(opts_pillar)
    opts_merged.update(opts)

    if profile:
        conf = opts_merged.get(profile, {})
    else:
        conf = opts_merged

    url = conf.get('vault.url', 'https://127.0.0.1:8200')
    token = conf.get('vault.token', '')
    cert_pem = conf.get('vault.cert_pem', '')
    key_pem = conf.get('vault.key_pem', '')
    verify = conf.get('vault.verify', True)

    if HAS_LIBS:
        return Vault(url=url, token=token, cert=(cert_pem, key_pem), verify=verify)
    else:
        raise CommandExecutionError(
            '(unable to import requests, '
            'module most likely not installed)'
        )


class Vault(object):
    def __init__(self, url=None, token=None, cert=None, verify=True):
        """
        Initialize the vault client

        # Using plaintext
        vault = Vault()
        vault = Vault(url='http://localhost:8200')
        vault = Vault(url='http://localhost:8200', token='dda4d-fdfdfd3-fdffd3-bhhnhn')

        # Using TLS
        vault = Vault(url='https://localhost:8200')

        # Using TLS with client-side certificate authentication
        vault = Vault(url='https://localhost:8200',
                      cert=('path/to/cert.pem', 'path/to/key.pem'))
        """
        self._url = url if url else 'http://localhost:8200'
        self._cert = cert
        self._verify = verify
        self.token = token
        return

    def transit_decrypt(self, name, ciphertext, **kwargs):
        """
        POST /transit/decrypt/<name>
        """
        params = {'ciphertext': ciphertext}

        r = self._post('/v1/transit/decrypt/{0}'.format(name), data=json.dumps(params))
        return base64.b64decode(r.json()['data']['plaintext'])

    def _post(self, url, **kwargs):
        return self.__request('post', url, **kwargs)

    def __request(self, method, url, headers=None, **kwargs):
        url = self._url + url

        if not headers:
            headers = {}

        if self.token:
            headers['X-Vault-Token'] = self.token

        response = requests.request(method,
                                    url,
                                    cert=self._cert,
                                    verify=self._verify,
                                    headers=headers,
                                    **kwargs)

        if response.status_code >= 400 and response.status_code < 600:
            errors = response.json().get('errors')

            if response.status_code == 400:
                raise InvalidRequest(errors=errors)
            elif response.status_code == 401:
                raise Unauthorized(errors=errors)
            elif response.status_code == 404:
                raise InvalidPath(errors=errors)
            elif response.status_code == 429:
                raise RateLimitExceeded(errors=errors)
            elif response.status_code == 500:
                raise InternalServerError(errors=errors)
            elif response.status_code == 503:
                raise VaultDown(errors=errors)
            else:
                raise UnknownError()
        return response

class VaultError(Exception):
    def __init__(self, message=None, errors=None):
        if errors:
            message = ', '.join(errors)

        self.errors = errors

        super(VaultError, self).__init__(message)

class InvalidRequest(VaultError):
    pass

class Unauthorized(VaultError):
    pass

class InvalidPath(VaultError):
    pass

class RateLimitExceeded(VaultError):
    pass

class InternalServerError(VaultError):
    pass

class VaultDown(VaultError):
    pass

class UnexpectedError(VaultError):
    pass

vault_renderer.py

# -*- coding: utf-8 -*-
'''
Renderer that will decrypt Vault ciphers using the Transit backend.

Any key in the SLS file can be a Vault cipher, and this renderer will decrypt
it before passing it off to Salt. This allows you to safely store secrets in
source control, in such a way that only your Salt master can decrypt them and
distribute them only to the minions that need them.

The typical use-case would be to use ciphers in your pillar data, and your 
encryption key will be on the Vault server. Developers will query the Vault
API to create a cipher text from plaintext. See the Vault docs for more
information, https://www.vaultproject.io/docs/secrets/transit/index.html.

This renderer requires the python-requests package.

In order to use an Vault server, a profile must be created in the master
configuration file:

Token example:

.. code-block:: yaml

    hashicorp_vault_config:
      url: https://127.0.0.1:8200
      verify: True
      keyname: saltstack
      token: 354r0488-9993-8545-140c-93af3c494as2

Client TLS Cert example:

.. code-block:: yaml

    hashicorp_vault_config:
      url: https://127.0.0.1:8200
      verify: True
      keyname: saltstack
      cert_pem: path/to/cert.pem
      key_pem: path/to/key.pem

Now that the configuration is done, Generate a encryption key. 
On your Vault sever, run:

.. code-block:: bash

    # vault write transit/keys/saltstack salt=stack

Now, to encrypt secrets, query the Vault server with a base64 encoding of your
string to receive a cipher text response:

.. code-block:: bash

    $ echo -n "the quick brown fox" | base64 | vault write transit/encrypt/saltstack plaintext=-

Set up the renderer on your master by adding something like this line to your
config:

.. code-block:: yaml

    renderer: jinja | yaml | vault

Now you can include your ciphers in your pillar data like so:

.. code-block:: yaml

    a-secret: vault:v0:czEwyKqGZY/limnuzDCUUe5AK0tbBObWqeZgFqxCuIqq7A84SeiOq3sKD0Y/KUvv
'''

# Import python libs
from __future__ import absolute_import
import os
import re
import json
import base64
import logging

# Import salt libs
import salt.utils
import salt.syspaths
from salt.exceptions import SaltRenderError

# Import 3rd-party libs
import salt.ext.six as six
# pylint: disable=import-error
try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False
# pylint: enable=import-error

from salt.exceptions import SaltRenderError

log = logging.getLogger(__name__)
VAULT_HEADER = re.compile(r'vault:v0:')


def decrypt_ciphertext(c, vault, keyname):
    '''
    Given a block of ciphertext as a string, and a vault object, try to decrypt
    the cipher and return the decrypted string. If the cipher cannot be
    decrypted, log the error, and return the ciphertext back out.
    '''
    decrypted_data = vault.transit_decrypt(keyname, c)
    if not decrypted_data:
        log.info("Could not decrypt cipher {0}, received {1}".format(
            c, decrypted_data))
        return c
    else:
        return str(decrypted_data)


def decrypt_object(o, vault, keyname):
    '''
    Recursively try to decrypt any object. If the object is a string, and
    it contains a valid Vault header, decrypt it, otherwise keep going until
    a string is found.
    '''
    if isinstance(o, str):
        if VAULT_HEADER.search(o):
            return decrypt_ciphertext(o, vault, keyname)
        else:
            return o
    elif isinstance(o, dict):
        for k, v in o.items():
            o[k] = decrypt_object(v, vault, keyname)
        return o
    elif isinstance(o, list):
        for number, value in enumerate(o):
            o[number] = decrypt_object(value, vault, keyname)
        return o
    else:
        return o


def render(vault_data, saltenv='base', sls='', argline='', **kwargs):
    '''
    Create a vault object for given vault settings, and then use it to try to 
    decrypt the data to be rendered.
    '''
    if not HAS_REQUESTS:
        raise SaltRenderError('Requests unavailable')
    if 'config.get' in __salt__:
        url = __salt__['config.get']('hashicorp_vault_config:url')
        verify = __salt__['config.get']('hashicorp_vault_config:verify', True)
        keyname = __salt__['config.get']('hashicorp_vault_config:keyname')
        token = __salt__['config.get']('hashicorp_vault_config:token')
        cert_pem = __salt__['config.get']('hashicorp_vault_config:cert_pem')
        key_pem = __salt__['config.get']('hashicorp_vault_config:key_pem')
    else:
        raise SaltRendererError('Cannot initialize Vault, no config parameters found')
    log.debug('HashiCorp Vault url: {0}'.format(url))
    log.debug('HashiCorp Vault verify: {0}'.format(verify))
    log.debug('HashiCorp Vault keyname: {0}'.format(keyname))
    log.debug('HashiCorp Vault token: {0}'.format(token))
    log.debug('HashiCorp Vault cert_pem: {0}'.format(cert_pem))
    log.debug('HashiCorp Vault key_pem: {0}'.format(key_pem))
    vault = Vault(url=url, token=token, cert=(cert_pem, key_pem), verify=verify)
    return decrypt_object(vault_data, vault, keyname)


class Vault(object):
    def __init__(self, url=None, token=None, cert=None, verify=True):
        """
        Initialize the vault client

        # Using plaintext
        vault = Vault()
        vault = Vault(url='http://localhost:8200')
        vault = Vault(url='http://localhost:8200', token='dda4d-fdfdfd3-fdffd3-bhhnhn')

        # Using TLS
        vault = Vault(url='https://localhost:8200')

        # Using TLS with client-side certificate authentication
        vault = Vault(url='https://localhost:8200',
                      cert=('path/to/cert.pem', 'path/to/key.pem'))
        """
        self._url = url if url else 'http://localhost:8200'
        self._cert = cert
        self._verify = verify
        self.token = token
        return

    def transit_decrypt(self, name, ciphertext, **kwargs):
        """
        POST /transit/decrypt/<name>
        """
        params = {'ciphertext': ciphertext}

        r = self._post('/v1/transit/decrypt/{0}'.format(name), data=json.dumps(params))
        return base64.b64decode(r.json()['data']['plaintext'])

    def _post(self, url, **kwargs):
        return self.__request('post', url, **kwargs)

    def __request(self, method, url, headers=None, **kwargs):
        url = self._url + url

        if not headers:
            headers = {}

        if self.token:
            headers['X-Vault-Token'] = self.token

        response = requests.request(method,
                                    url,
                                    cert=self._cert,
                                    verify=self._verify,
                                    headers=headers,
                                    **kwargs)

        if response.status_code >= 400 and response.status_code < 600:
            errors = response.json().get('errors')

            if response.status_code == 400:
                raise InvalidRequest(errors=errors)
            elif response.status_code == 401:
                raise Unauthorized(errors=errors)
            elif response.status_code == 404:
                raise InvalidPath(errors=errors)
            elif response.status_code == 429:
                raise RateLimitExceeded(errors=errors)
            elif response.status_code == 500:
                raise InternalServerError(errors=errors)
            elif response.status_code == 503:
                raise VaultDown(errors=errors)
            else:
                raise UnknownError()
        return response

class VaultError(Exception):
    def __init__(self, message=None, errors=None):
        if errors:
            message = ', '.join(errors)

        self.errors = errors

        super(VaultError, self).__init__(message)

class InvalidRequest(VaultError):
    pass

class Unauthorized(VaultError):
    pass

class InvalidPath(VaultError):
    pass

class RateLimitExceeded(VaultError):
    pass

class InternalServerError(VaultError):
    pass

class VaultDown(VaultError):
    pass

class UnexpectedError(VaultError):
    pass

Excellent work !!!

Couple of questions:

• I get the flow which is exactly like the current GPG render except this time we are using Vault but my question is:

_the vault_module.py will live on modules dir on the salt master but where will the renderer live?