import_pillar2mongo.py

#!/usr/bin/env python


import pprint
import urllib
import salt.client
import salt.config
import sys
from pymongo import MongoClient

if len(sys.argv) < 2:
    sys.exit('Usage: %s pillar1 [pillar2 pillar3...]' % sys.argv[0])


pp = pprint.PrettyPrinter(indent=4)
# this dictionnary is used for '.' and '$' special chars
# these characters are not allowed in mongodb keys
subs = { '.': 'U+002E', \
         '$': 'U+0024' }


config_file = '/etc/salt/master'
master_opts = salt.config.client_config(config_file)




def replace_special_chars(pillar_content):
    new = {}
    if isinstance(pillar_content, dict):
        for key, val in pillar_content.iteritems():
            found = False
            for symbol in subs.iterkeys():
                if symbol in key:
                    found = True
                    newkey = key.replace(symbol, subs[symbol])
                    newkey = newkey.encode('UTF-8')
                    new[newkey] = val
                    if isinstance(val, dict) or isinstance(val, list):
                        replace_special_chars(val)
            if found is False:
                new[key] = val
            found = False
    elif isinstance(pillar_content, list):
        for item in pillar_content:
            if isinstance(item, dict):
                replace_special_chars(item)
    return(new)



def main():
    """
    This code import pillars to minions

    It will import each sub-key as mongo '_id'.
    So, basically, it will work if each first sub-keys of the pillar
    are valid minions id.

    If you want to import all a content of a pillar node
    to a minion use "import_pillar2mongo_on_master.py" instead
    and change the "tgt_id" to the minion id you want.
    """

    # https://api.mongodb.com/python/current/examples/authentication.html
    Mechanism = 'MONGODB-CR'
    #Mechanism='SCRAM-SHA-1'

    # mongodb connection
    if 'mongo.user' in master_opts:
        User = master_opts['mongo.user']
    else:
        User = ''
    if 'mongo.password' in master_opts:
        Password = master_opts['mongo.password']
    else:
        Password = ''
    if 'mongo.host' in master_opts:
        Host = master_opts['mongo.host']
    else:
        Host = 'localhost'
    if 'mongo.port' in master_opts:
        Port = master_opts['mongo.port']
    else:
        Port = '27017'

    db_name = 'salt'
    col_name = 'pillar'

    if Password:
        Password = urllib.quote_plus(Password)
        uri = "mongodb://" + User + ":" + Password + "@" + \
        Host + ':' + str(Port) + "/" + db_name + "?authMechanism=" + \
        Mechanism
        print("Connecting to %s" % (uri))
        client = MongoClient(uri)
    else:
        client = MongoClient('mongodb://' + Host + ':' + Port + '/')

    db = client[db_name]
    pillars = db[col_name]

    caller = salt.client.Caller()

    for pillar_name in sys.argv:
        if pillar_name is not sys.argv[0]:
            pillar_content = caller.function('pillar.get', pillar_name)
            pillar_content = replace_special_chars(pillar_content)
            if isinstance(pillar_content, dict):
                for _id, value in pillar_content.iteritems():
                    print("\n################\nFollowing pillar will be " + \
                    "inserted: \n")
                    pp.pprint(_id)
                    pp.pprint(value)
                    pillars.insert({ '_id': _id, 'mongo_pillar': value })
            #elif isinstance(pillar_content, list):
            #    for item in pillar_content:
            #        if isinstance(item, dict):


    #listing collections
    collections = db.collection_names(include_system_collections=False)
    print("Your collections")
    pp.pprint(collections)

if __name__ == "__main__":
    sys.exit(main())

import_pillar2mongo_on_master.py

#!/usr/bin/env python


import pprint
import urllib
import salt.client
import salt.config
import sys
from pymongo import MongoClient

if len(sys.argv) < 2:
    sys.exit('Usage: %s pillar1 [pillar2 pillar3...]' % sys.argv[0])


pp = pprint.PrettyPrinter(indent=4)
# this dictionnary is used for '.' and '$' special chars
# these characters are not allowed in mongodb keys
subs = { '.': 'U+002E', \
         '$': 'U+0024' }


master_conf_file = '/etc/salt/master'
minion_conf_file = '/etc/salt/minion'
master_opts = salt.config.client_config(master_conf_file)
__opts__ = salt.config.minion_config('/etc/salt/minion')

# If it is launched from master, you can retrieve master id with
# local minion id (useful if it is different from hostname)
master_id = __opts__['id']


def replace_special_chars(pillar_content):
    new = {}
    if isinstance(pillar_content, dict):
        for key, val in pillar_content.iteritems():
            found = False
            for symbol in subs.iterkeys():
                if symbol in key:
                    found = True
                    newkey = key.replace(symbol, subs[symbol])
                    newkey = newkey.encode('UTF-8')
                    new[newkey] = val
                    if isinstance(val, dict) or isinstance(val, list):
                        replace_special_chars(val)
            if found is False:
                new[key] = val
            found = False
    elif isinstance(pillar_content, list):
        for item in pillar_content:
            if isinstance(item, dict):
                replace_special_chars(item)
    return(new)



def main():
    """
    This code import all the pillar to the master minions (including
    the first key)

    If you want to import all a content of a pillar node
    to a minion change the "tgt_id" to the minion id you want.
    """
    # https://api.mongodb.com/python/current/examples/authentication.html
    Mechanism = 'MONGODB-CR'
    #Mechanism='SCRAM-SHA-1'

    # mongodb connection
    if 'mongo.user' in master_opts:
        User = master_opts['mongo.user']
    else:
        User = ''
    if 'mongo.password' in master_opts:
        Password = master_opts['mongo.password']
    else:
        Password = ''
    if 'mongo.host' in master_opts:
        Host = master_opts['mongo.host']
    else:
        Host = 'localhost'
    if 'mongo.port' in master_opts:
        Port = master_opts['mongo.port']
    else:
        Port = '27017'

    db_name = 'salt'
    col_name = 'pillar'

    if Password:
        Password = urllib.quote_plus(Password)
        uri = "mongodb://" + User + ":" + Password + "@" + \
        Host + ':' + str(Port) + "/" + db_name + "?authMechanism=" + \
        Mechanism
        print("Connecting to %s" % (uri))
        client = MongoClient(uri)
    else:
        client = MongoClient('mongodb://' + Host + ':' + Port + '/')

    db = client[db_name]
    pillars = db[col_name]

    caller = salt.client.Caller()

    for pillar_name in sys.argv:
        if pillar_name is not sys.argv[0]:
            pillar_content = caller.function('pillar.get', pillar_name)
            pillar_content = replace_special_chars(pillar_content)
            # overwrite tgt_id if you need another minion
            tgt_id = master_id
            print("\n################\nFollowing pillar will be " + \
            "inserted: \n")
            pp.pprint("_id: " + tgt_id)
            pp.pprint(pillar_name + ": " + repr(pillar_content))
            pillars.insert({'_id': tgt_id, pillar_name: pillar_content})

    #listing collections
    collections = db.collection_names(include_system_collections=False)
    print("Your collections")
    pp.pprint(collections)

if __name__ == "__main__":
    sys.exit(main())

2 scripts to import SLS/Yaml files (pillars) from SaltStack to mongodb/JSON

With these two scripts, you can either import pillars directly to minions (*) or an entire pillar content to a single minion (**)/master.

(*) each first sub-keys of the pillar is considered as the minion id (mongo '_id' field).
(**) By using the second script and then overwrite "tgt_id" which is the master id by default.