Skip to content

Instantly share code, notes, and snippets.

@ryancurrah
Created December 18, 2015 01:25
Show Gist options
  • Save ryancurrah/40aefb2baa8a69522127 to your computer and use it in GitHub Desktop.
Save ryancurrah/40aefb2baa8a69522127 to your computer and use it in GitHub Desktop.
SaltStack Module and Renderer for HashiCorp Vault
# -*- coding: utf-8 -*-
'''
Execution module to work with HashiCorp's Vault
:depends: - python-requests
In order to use an this module, a profile must be created in the master
configuration file:
Token example:
.. code-block:: yaml
hashicorp_vault_config:
url: https://127.0.0.1:8200
verify: True
keyname: saltstack
token: 674r0488-9993-8545-140c-93af3c494at9
Client TLS Cert example:
.. code-block:: yaml
hashicorp_vault_config:
url: https://127.0.0.1:8200
verify: True
keyname: saltstack
cert_pem: path/to/cert.pem
key_pem: path/to/key.pem
Now that the configuration is done, Generate a encryption key.
On your Vault sever, run:
.. code-block:: bash
# vault write transit/keys/saltstack salt=stack
Now, to encrypt secrets, query the Vault server with a base64 encoding of your
string to receive a cipher text response:
.. code-block:: bash
$ echo -n "the quick brown fox" | base64 | vault write transit/encrypt/saltstack plaintext=-
Set up the renderer on your master by adding something like this line to your
config:
.. code-block:: yaml
renderer: jinja | yaml | vault
Now you can include your ciphers in your pillar data like so:
.. code-block:: yaml
a-secret: vault:v0:czEwyKqGZY/limnuzDCUUe5AK0tbBObWqeZgFqxCuIqq7A84SeiOq3sKD0Y/KUvv
'''
from __future__ import absolute_import
# Import python libs
import json
import base64
import logging
# Import salt libs
from salt.exceptions import CommandExecutionError
# Import 3rd-party libs
import salt.ext.six as six
# pylint: disable=import-error
try:
import requests
HAS_LIBS = True
except ImportError:
HAS_LIBS = False
# pylint: enable=import-error
__virtualname__ = 'vault'
log = logging.getLogger(__name__)
__func_alias__ = {
'get_': 'get',
}
def __virtual__():
'''
Only return if python-requests is installed
'''
return __virtualname__ if HAS_LIBS else False
def get_(keyname, ciphertext, profile=None):
'''
.. versionadded:: 2015.7.0
Get a value from vault transit backend, by key
CLI Examples:
.. code-block:: bash
salt myminion vault.get keyname vault:v0:czEwyKqGZY/limnuzDCUUe5AK0tbBObWqeZgFqxCuIqq7A84SeiOq3sKD0Y/KUvv
'''
client = get_conn(__opts__, profile)
return client.transit_decrypt(keyname, ciphertext)
def get_conn(opts, profile=None):
'''
.. versionadded:: 2014.15.0
Return a client object for accessing vault
'''
opts_pillar = opts.get('pillar', {})
opts_master = opts_pillar.get('master', {})
opts_merged = {}
opts_merged.update(opts_master)
opts_merged.update(opts_pillar)
opts_merged.update(opts)
if profile:
conf = opts_merged.get(profile, {})
else:
conf = opts_merged
url = conf.get('vault.url', 'https://127.0.0.1:8200')
token = conf.get('vault.token', '')
cert_pem = conf.get('vault.cert_pem', '')
key_pem = conf.get('vault.key_pem', '')
verify = conf.get('vault.verify', True)
if HAS_LIBS:
return Vault(url=url, token=token, cert=(cert_pem, key_pem), verify=verify)
else:
raise CommandExecutionError(
'(unable to import requests, '
'module most likely not installed)'
)
class Vault(object):
def __init__(self, url=None, token=None, cert=None, verify=True):
"""
Initialize the vault client
# Using plaintext
vault = Vault()
vault = Vault(url='http://localhost:8200')
vault = Vault(url='http://localhost:8200', token='dda4d-fdfdfd3-fdffd3-bhhnhn')
# Using TLS
vault = Vault(url='https://localhost:8200')
# Using TLS with client-side certificate authentication
vault = Vault(url='https://localhost:8200',
cert=('path/to/cert.pem', 'path/to/key.pem'))
"""
self._url = url if url else 'http://localhost:8200'
self._cert = cert
self._verify = verify
self.token = token
return
def transit_decrypt(self, name, ciphertext, **kwargs):
"""
POST /transit/decrypt/<name>
"""
params = {'ciphertext': ciphertext}
r = self._post('/v1/transit/decrypt/{0}'.format(name), data=json.dumps(params))
return base64.b64decode(r.json()['data']['plaintext'])
def _post(self, url, **kwargs):
return self.__request('post', url, **kwargs)
def __request(self, method, url, headers=None, **kwargs):
url = self._url + url
if not headers:
headers = {}
if self.token:
headers['X-Vault-Token'] = self.token
response = requests.request(method,
url,
cert=self._cert,
verify=self._verify,
headers=headers,
**kwargs)
if response.status_code >= 400 and response.status_code < 600:
errors = response.json().get('errors')
if response.status_code == 400:
raise InvalidRequest(errors=errors)
elif response.status_code == 401:
raise Unauthorized(errors=errors)
elif response.status_code == 404:
raise InvalidPath(errors=errors)
elif response.status_code == 429:
raise RateLimitExceeded(errors=errors)
elif response.status_code == 500:
raise InternalServerError(errors=errors)
elif response.status_code == 503:
raise VaultDown(errors=errors)
else:
raise UnknownError()
return response
class VaultError(Exception):
def __init__(self, message=None, errors=None):
if errors:
message = ', '.join(errors)
self.errors = errors
super(VaultError, self).__init__(message)
class InvalidRequest(VaultError):
pass
class Unauthorized(VaultError):
pass
class InvalidPath(VaultError):
pass
class RateLimitExceeded(VaultError):
pass
class InternalServerError(VaultError):
pass
class VaultDown(VaultError):
pass
class UnexpectedError(VaultError):
pass
# -*- coding: utf-8 -*-
'''
Renderer that will decrypt Vault ciphers using the Transit backend.
Any key in the SLS file can be a Vault cipher, and this renderer will decrypt
it before passing it off to Salt. This allows you to safely store secrets in
source control, in such a way that only your Salt master can decrypt them and
distribute them only to the minions that need them.
The typical use-case would be to use ciphers in your pillar data, and your
encryption key will be on the Vault server. Developers will query the Vault
API to create a cipher text from plaintext. See the Vault docs for more
information, https://www.vaultproject.io/docs/secrets/transit/index.html.
This renderer requires the python-requests package.
In order to use an Vault server, a profile must be created in the master
configuration file:
Token example:
.. code-block:: yaml
hashicorp_vault_config:
url: https://127.0.0.1:8200
verify: True
keyname: saltstack
token: 354r0488-9993-8545-140c-93af3c494as2
Client TLS Cert example:
.. code-block:: yaml
hashicorp_vault_config:
url: https://127.0.0.1:8200
verify: True
keyname: saltstack
cert_pem: path/to/cert.pem
key_pem: path/to/key.pem
Now that the configuration is done, Generate a encryption key.
On your Vault sever, run:
.. code-block:: bash
# vault write transit/keys/saltstack salt=stack
Now, to encrypt secrets, query the Vault server with a base64 encoding of your
string to receive a cipher text response:
.. code-block:: bash
$ echo -n "the quick brown fox" | base64 | vault write transit/encrypt/saltstack plaintext=-
Set up the renderer on your master by adding something like this line to your
config:
.. code-block:: yaml
renderer: jinja | yaml | vault
Now you can include your ciphers in your pillar data like so:
.. code-block:: yaml
a-secret: vault:v0:czEwyKqGZY/limnuzDCUUe5AK0tbBObWqeZgFqxCuIqq7A84SeiOq3sKD0Y/KUvv
'''
# Import python libs
from __future__ import absolute_import
import os
import re
import json
import base64
import logging
# Import salt libs
import salt.utils
import salt.syspaths
from salt.exceptions import SaltRenderError
# Import 3rd-party libs
import salt.ext.six as six
# pylint: disable=import-error
try:
import requests
HAS_REQUESTS = True
except ImportError:
HAS_REQUESTS = False
# pylint: enable=import-error
from salt.exceptions import SaltRenderError
log = logging.getLogger(__name__)
VAULT_HEADER = re.compile(r'vault:v0:')
def decrypt_ciphertext(c, vault, keyname):
'''
Given a block of ciphertext as a string, and a vault object, try to decrypt
the cipher and return the decrypted string. If the cipher cannot be
decrypted, log the error, and return the ciphertext back out.
'''
decrypted_data = vault.transit_decrypt(keyname, c)
if not decrypted_data:
log.info("Could not decrypt cipher {0}, received {1}".format(
c, decrypted_data))
return c
else:
return str(decrypted_data)
def decrypt_object(o, vault, keyname):
'''
Recursively try to decrypt any object. If the object is a string, and
it contains a valid Vault header, decrypt it, otherwise keep going until
a string is found.
'''
if isinstance(o, str):
if VAULT_HEADER.search(o):
return decrypt_ciphertext(o, vault, keyname)
else:
return o
elif isinstance(o, dict):
for k, v in o.items():
o[k] = decrypt_object(v, vault, keyname)
return o
elif isinstance(o, list):
for number, value in enumerate(o):
o[number] = decrypt_object(value, vault, keyname)
return o
else:
return o
def render(vault_data, saltenv='base', sls='', argline='', **kwargs):
'''
Create a vault object for given vault settings, and then use it to try to
decrypt the data to be rendered.
'''
if not HAS_REQUESTS:
raise SaltRenderError('Requests unavailable')
if 'config.get' in __salt__:
url = __salt__['config.get']('hashicorp_vault_config:url')
verify = __salt__['config.get']('hashicorp_vault_config:verify', True)
keyname = __salt__['config.get']('hashicorp_vault_config:keyname')
token = __salt__['config.get']('hashicorp_vault_config:token')
cert_pem = __salt__['config.get']('hashicorp_vault_config:cert_pem')
key_pem = __salt__['config.get']('hashicorp_vault_config:key_pem')
else:
raise SaltRendererError('Cannot initialize Vault, no config parameters found')
log.debug('HashiCorp Vault url: {0}'.format(url))
log.debug('HashiCorp Vault verify: {0}'.format(verify))
log.debug('HashiCorp Vault keyname: {0}'.format(keyname))
log.debug('HashiCorp Vault token: {0}'.format(token))
log.debug('HashiCorp Vault cert_pem: {0}'.format(cert_pem))
log.debug('HashiCorp Vault key_pem: {0}'.format(key_pem))
vault = Vault(url=url, token=token, cert=(cert_pem, key_pem), verify=verify)
return decrypt_object(vault_data, vault, keyname)
class Vault(object):
def __init__(self, url=None, token=None, cert=None, verify=True):
"""
Initialize the vault client
# Using plaintext
vault = Vault()
vault = Vault(url='http://localhost:8200')
vault = Vault(url='http://localhost:8200', token='dda4d-fdfdfd3-fdffd3-bhhnhn')
# Using TLS
vault = Vault(url='https://localhost:8200')
# Using TLS with client-side certificate authentication
vault = Vault(url='https://localhost:8200',
cert=('path/to/cert.pem', 'path/to/key.pem'))
"""
self._url = url if url else 'http://localhost:8200'
self._cert = cert
self._verify = verify
self.token = token
return
def transit_decrypt(self, name, ciphertext, **kwargs):
"""
POST /transit/decrypt/<name>
"""
params = {'ciphertext': ciphertext}
r = self._post('/v1/transit/decrypt/{0}'.format(name), data=json.dumps(params))
return base64.b64decode(r.json()['data']['plaintext'])
def _post(self, url, **kwargs):
return self.__request('post', url, **kwargs)
def __request(self, method, url, headers=None, **kwargs):
url = self._url + url
if not headers:
headers = {}
if self.token:
headers['X-Vault-Token'] = self.token
response = requests.request(method,
url,
cert=self._cert,
verify=self._verify,
headers=headers,
**kwargs)
if response.status_code >= 400 and response.status_code < 600:
errors = response.json().get('errors')
if response.status_code == 400:
raise InvalidRequest(errors=errors)
elif response.status_code == 401:
raise Unauthorized(errors=errors)
elif response.status_code == 404:
raise InvalidPath(errors=errors)
elif response.status_code == 429:
raise RateLimitExceeded(errors=errors)
elif response.status_code == 500:
raise InternalServerError(errors=errors)
elif response.status_code == 503:
raise VaultDown(errors=errors)
else:
raise UnknownError()
return response
class VaultError(Exception):
def __init__(self, message=None, errors=None):
if errors:
message = ', '.join(errors)
self.errors = errors
super(VaultError, self).__init__(message)
class InvalidRequest(VaultError):
pass
class Unauthorized(VaultError):
pass
class InvalidPath(VaultError):
pass
class RateLimitExceeded(VaultError):
pass
class InternalServerError(VaultError):
pass
class VaultDown(VaultError):
pass
class UnexpectedError(VaultError):
pass
@DanyC97
Copy link

DanyC97 commented Dec 19, 2015

Excellent work !!!

Couple of questions:

  • I get the flow which is exactly like the current GPG render except this time we are using Vault but my question is:

_the vault_module.py will live on modules dir on the salt master but where will the renderer live?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment