Saltstack pillar data assignment based on structured minion ID

pillar.sls

#!py

import re

# Some hosts have legacy names so remap them here
known_hosts = {
        r'^fmfa([0-9]+)($|\.)' : 'gr-prod-f-app-%1',
        r'^fmpa([0-9]+)($|\.)' : 'gr-prod-p-app-%1',
        }

standard = """
    (?P<loc>[a-z0-9]+)-  # Location part
    (?P<env>[a-z0-9]+)-  # Environment part
    (?P<cat>[fpc])-      # Category
    (?P<type>[a-z0-9]+)- # Machine type part
    (?P<num>[0-9]{1,2})  # Number part
    """

standard_regex = re.compile(standard, re.IGNORECASE | re.VERBOSE)

cat_map = { 'f': 'free', 'c': 'common', 'p': 'premium', 'z' : 'custdmz' }
loc_name_map = { 
        'sal': 'Salisbury',
        'lon' : 'London',
        'aws' : 'Amazon Web Services'
        }

# List full environments where beta stuff should be added in parellel
envs_with_beta = [ 'aws-dev' ]

def run():
    hostname = __grains__['id']

    # Deal with existing hosts by mapping their names to a standard form
    hostname = map_known(hostname)

    pillar = match_general(hostname)

    if pillar:
        # Record the canonical name in the pillar
        # If the machine was a "known host" then this will be the name
        # we mapped it to
        pillar['canonical_name'] = hostname

        # Add the list of included SLS files
        add_includes(pillar)

    return pillar

def map_known(hostname):

    for key in known_hosts.keys():
        match = re.match(key, hostname, re.I)
        if match:
            return replace_groups(known_hosts[key], match)

    return hostname

def match_general(hostname):
    result = {}

    match = standard_regex.match(hostname)
    if match:

        # Now we have loc, cat, env, type, num

        # First, turn cat into a word, e.g. f -> free
        result['category'] = cat_map[match.group('cat')]

        # Keep location and environment unmodified
        result['location'] = match.group('loc')
        result['environment'] = match.group('env')

        # Add a key for the full location name
        result['location_full_name'] = loc_name_map.get(result['location'],
                result['location'])

        # Full env is loc + env
        result['full_env'] = result['location'] + '-' + result['environment']

        # Beta ?
        result['include_beta'] = result['full_env'] in envs_with_beta

        # Store the type field
        result['type'] = match.group('type') 

        # Primary role is just the type field, initially.
        # E.g. 'nginx'
        result['primary_role'] = match.group('type') 

        # Add the number field to pillar too (might be useful)
        result['num_in_pool'] = match.group('num')

        # We have some special case handling
        # This might decorate the primary role, e.g. 'app' -> 'app-1'
        special_cases(result, int(match.group('num')))

        # Roles array is the main role, e.g. iis, plus the same decorated with
        # the category, e.g [ 'iis', 'free-iis' ]
        # If the primary role has been decorated, those will be added too, e.g.
        # [ 'db', 'free-db', 'db-master', 'free-db-master' ]

        result['roles'] = list(set([
            result['type'], 
            result['category'] + '-' + result['type'],
            result['primary_role'], 
            result['category'] + '-' + result['primary_role']
            ]))

        # Finally, the primary role should include the category too
        result['primary_role'] = result['category'] + '-' + result['primary_role']

    return result

def special_cases(pillar, machine_number):

    # For each app server, the number is part of the role
    if pillar['primary_role'] == 'app':
        pillar['primary_role'] = 'app-%d' % machine_number

    # For database and dns servers, number 1 is a master and the rest are slaves
    if pillar['primary_role'] in ['db','dns']:
        if machine_number == 1:
            pillar['primary_role'] = '%s-master' % pillar['primary_role']
        else:
            pillar['primary_role'] = '%s-slave' % pillar['primary_role']

    # For nginx servers, number 1 is a primary one
    if pillar['primary_role'] == 'nginx':
        if machine_number == 1:
            pillar['primary_role'] = 'nginx-primary'

def replace_groups(text, regex_match):
    """
    Replace %1..%9 placeholders in text with
    groups from the provided match object
    """

    def get_replacement(m):
        # Match group 1 contains the number from the "%n" string
        ix = int(m.group(1))
        return regex_match.group(ix)

    return re.sub(r'%([0-9])', get_replacement, text)

def add_includes(pillar):

    # Includes are based on roles and environments

    # We tend to include a smaller number of files based on more specific data
    # since those files if required can include shared general case data
    # If we didn't do this you'd have to create a lot of empty sls files just
    # to avoid errors

    # Include based on primary role, e.g. 'free-nginx' -> 'role.free.nginx'
    pillar['include'] = [ 'role.' + pillar['primary_role'].replace('-','.') ]

    # Include based on location and environment
    # e.g. 'bs' and 'prod' -> 'env.bs.prod'

    env_include = '.'.join([ 'env', pillar['location'], pillar['environment'] ]) 

    pillar['include'].append(env_include) 

    # Include beta env if required
    if pillar['include_beta']:
        pillar['include'].append(env_include + '.beta') 

    pillar['debug_include'] = pillar['include']

    return pillar

top.sls

base:

  # ---------------------------------------------
  # Select by OS

  'G@os:Ubuntu or G@os:Debian':
    - match: compound
    - os.linux

  'os:Windows':
    - match: grain
    - os.windows

  '*':
    - pillar
    - all
    - saltmine