Ansible CloudStack Provisioning with libcloud

Ansible CloudStack Provisioning with libcloud

{'ex_keyname': 'jmoody_fifthecho', 'image': <NodeImage: id=85d5a794-c3f4-49fd-afee-19132a850f79, name=Datapipe: CentOS 6 (64-bit), driver=CloudStack  ...>, 'location': <NodeLocation: id=5, name=New York Metro, country=Unknown, driver=CloudStack>, 'ex_security_groups': ['default', 'TestSG'], 'size': <NodeSize: id=44, name=kilo-2-20, ram=2048 disk=0 bandwidth=0 price=0 driver=CloudStack ...>}
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/home/jmoody/.ansible/tmp/ansible-tmp-1399056893.0-119747009807487/cloudstack in <module>()
   2544         return '%.2f %s' % (float(size)/ limit, suffix)
   2545 
-> 2546 main()
   2547 

/home/jmoody/.ansible/tmp/ansible-tmp-1399056893.0-119747009807487/cloudstack in main()
    352     connection = initialize_connection(module)
    353     if module.params['state'] == 'present':
--> 354         instance = create_instances(connection, module)
    355         print "Instance " + instance['id']
    356 

/home/jmoody/.ansible/tmp/ansible-tmp-1399056893.0-119747009807487/cloudstack in create_instances(connection, instance)
    327     provisioning_parameters['ex_keyname'] = instance_ssh_key
    328     print provisioning_parameters
--> 329     return connection.create_node(provisioning_parameters)
    330 
    331 

TypeError: create_node() takes exactly 1 argument (2 given)

cloudstack.py

#!/usr/bin/env python

# (c) 2014, Jeff Moody <fifthecho@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#####################################################################


DOCUMENTATION = '''
---
module: CloudStack
short_description: create / delete / stop / restart an instance in an Apache CloudStack Cloud
description:
     - creates / deletes an Apache CloudStack Cloud instance and optionally waits for it to be 'running'.
version_added: "1.3"
options:
  state:
    description:
     - Indicate desired state of the resource
    choices: ['present', 'active', 'started', 'absent', 'deleted', 'stopped', 'restarted']
    default: present
  api_key:
    description:
     - CloudStack API key
    default: null
  secret_key:
    description:
     - CloudStack API Secret key
  display_name:
    description:
     - Name to give the instance (alphanumeric, dashes, underscore)
    default: null
    type: string
  instance_id:
    description:
     - Unique ID of an Apache CloudStack instance
    aliases: lid
    default: null
    type: string
  ssh_pub_key:
    description:
     - SSH public key applied to root user
    default: null
    type: string
  templateid:
    description:
     - Template to be used when provisioning instance
    default: null
    type: string
  zone:
    description:
     - CloudStack Zone to provision server in
    default: null
    type: integer
  wait:
    description:
     - wait for the instance to be in state 'running' before returning
    default: "yes"
    choices: [ "yes", "no" ]
  wait_timeout:
    description:
     - how long before wait gives up, in seconds
    default: 300
requirements: [ "libcloud", "cloudmonkey" ]
author: Jeff Moody (fifthecho@gmail.com)
notes:
'''

EXAMPLES = '''
'''

import sys
import time
import os
import ConfigParser
import os.path

try:
    import json
except ImportError:
    import simplejson as json

try:
    import libcloud
    from libcloud.compute.types import Provider
    from libcloud.compute.providers import get_driver
    import libcloud.security as sec

    cloudstackconnection = get_driver(Provider.CLOUDSTACK)
    from ansible.module_utils.basic import *
except ImportError:
    print "failed=True msg='libcloud required for this module'"
    sys.exit(1)


def read_cloudstack_ini_settings():
    ''' Reads the settings from the cloudstack.ini file '''

    config = ConfigParser.SafeConfigParser()
    cloudstack_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cloudstack.ini')
    cloudstack_ini_path = os.environ.get('CLOUDSTACK_INI_PATH', cloudstack_default_ini_path)
    config.read(cloudstack_ini_path)

    if not config.has_section('driver'):
        raise ValueError('cloudstack.ini file must contain a [driver] section')

    if config.has_option('driver', 'access_key'):
        access_key = config.get('driver', 'access_key')
    else:
        raise ValueError('cloudstack.ini does not have an access key defined')

    if config.has_option('driver', 'secret_key'):
        secret_key = config.get('driver', 'secret_key')
    else:
        raise ValueError('cloudstack.ini does not have a secret key defined')

    if config.has_option('driver', 'url'):
        url = config.get('driver', 'url')
    else:
        raise ValueError('cloudstack.ini does not have a URL defined')

    return {'key': access_key, 'secret': secret_key, 'url': url}


def read_cloudmonkey_config_settings():
    config = ConfigParser.SafeConfigParser()
    cloudmonkey_default_config_path = os.path.join(os.path.expanduser('~'), '.cloudmonkey', 'config')
    cloudmonkey_ini_path = os.environ.get('CLOUDMONKEY_CONFIG_PATH', cloudmonkey_default_config_path)
    config.read(cloudmonkey_ini_path)
    if not config.has_section('user'):
        raise ValueError('CloudMonkey config must contain a [user] section')

    if config.has_option('user', 'apikey'):
        access_key = config.get('user', 'apikey')
    else:
        raise ValueError('CloudMonkey config does not have an apikey defined')

    if config.has_option('user', 'secretkey'):
        secret_key = config.get('user', 'secretkey')
    else:
        raise ValueError('CloudMonkey config does not have a secretkey defined')

    if not config.has_section('server'):
        raise ValueError('CloudMonkey config must contain a [server] section')

    if config.has_option('server', 'protocol'):
        url = config.get('server', 'protocol')
        url += "://"
    else:
        raise ValueError('CloudMonkey config does not have a protocol defined')

    if config.has_option('server', 'host'):
        url += config.get('server', 'host')
    else:
        raise ValueError('CloudMonkey config does not have a host defined')

    if config.has_option('server', 'port'):
        url += ":"
        url += config.get('server', 'port')
    else:
        raise ValueError('CloudMonkey config does not have a port defined')

    if config.has_option('server', 'path'):
        url += config.get('server', 'path')
    else:
        raise ValueError('CloudMonkey config does not have a path defined')

    return {'key': access_key, 'secret': secret_key, 'url': url}


def initialize_connection(instance):
    '''
    Initialize the libcloud connection to CloudStack, preferring Ansible overrides, then environment variables,
    then a local INI file, then the CloudMonkey config.
    '''
    if (instance.params['access_key'] is not None) and (instance.params['secret_key' is not NONE]):
        access_id = instance.params['access_key']
        secret_key = instance.params['secret_key']
        cloudstack_url = instance.params['api_url']
    elif (os.environ.get('CLOUDSTACK_ACCESS_KEY') is not None) and \
            (os.environ.get('CLOUDSTACK_SECRET_KEY') is not None) and (os.environ.get('CLOUDSTACK_URL') is not None):
        access_id = os.environ['CLOUDSTACK_ACCESS_KEY']
        secret_key = os.environ['CLOUDSTACK_SECRET_KEY']
        cloudstack_url = os.environ['CLOUDSTACK_URL']
    elif (os.environ.get('CLOUDSTACK_INI_PATH') is not None) or \
         (os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cloudstack.ini'))):
        settings = read_cloudstack_ini_settings()
        access_id = settings['key']
        secret_key = settings['secret']
        cloudstack_url = settings['url']
    elif (os.environ.get('CLOUDMONKEY_CONFIG_PATH') is not None) or \
         (os.path.isfile(os.path.join(os.path.expanduser('~'), '.cloudmonkey', 'config'))):
        settings = read_cloudmonkey_config_settings()
        access_id = settings['key']
        secret_key = settings['secret']
        cloudstack_url = settings['url']
    else:
        raise ValueError('CloudStack connection parameters not specified. Please use a cloudstack.ini '
                         'or specify your URL, Access Key, and Secret Key.')
    instance.params['access_key'] = access_id
    instance.params['secrety_key'] = secret_key
    instance.params['api_url'] = cloudstack_url
    return cloudstackconnection(key=access_id, secret=secret_key, url=cloudstack_url)


def list_sizes(connection):
    return connection.list_sizes()


def account_images(connection):
    return connection.list_images()


def security_group_list(connection):
    return connection.ex_list_security_groups()


def network_list(connection):
    return connection.ex_list_networks()


def account_keypairs(connection):
    return connection.list_key_pairs()


def zone_list(connection):
    return connection.list_locations()


def find_json(items, item_to_find, key):
    for item in items:
        key_value = unicode(item_to_find)
        if item[key] == key_value:
            return True

def find_object(items, item_to_find):
    for item in items:
        if item.id == item_to_find:
            return True

def find_sg_name(items, item_to_find):
    for item in items:
        key_value = unicode(item_to_find)
        if item['id'] == key_value:
            return str(item['name'])


def create_instances(connection, instance):
    '''
    Creates new instances
    '''

    '''
    Run tests to validate deployment information as valid.
    '''
    provisioning_parameters = {}
    cloud_templates = account_images(connection)
    service_offerings = list_sizes(connection)
    cloud_zones = zone_list(connection)
#    instance_keys = account_keypairs(connection)
    instance_template = instance.params['template_id']
    instance_service_offering = instance.params['size_id']
    instance_networks = instance.params['networks']
    instance_security_groups = instance.params['security_groups']
    instance_ssh_key = instance.params['key_name']
    instance_zone_id = instance.params['zone_id']
    flat_network = False

    if find_object(cloud_templates, instance_template) is False:
        raise ValueError(
            'Template ID not found. Please verify the template ID or that you have access to that template.')
    else:
        provisioning_parameters['image'] = [i for i in cloud_templates if i.id == instance_template][0]

    if find_object(service_offerings, instance_service_offering) is False:
        raise ValueError('Size ID not found. Please verify the Service Offering ID.')
    else:
        provisioning_parameters['size'] = [s for s in service_offerings if s.id == instance_service_offering][0]

    if find_object(cloud_zones, instance_zone_id) is False:
        raise ValueError('Zone ID not found. Please verify the Zone ID.')
    else:
        provisioning_parameters['location'] = [z for z in cloud_zones if z.id == instance_zone_id][0]

    if instance_networks is not None:
        provisioning_parameters['networks'] = []
        account_networks = network_list(connection)
        instance_networks = instance_networks.split(',')
        for network in instance_networks:
            if find_object(account_networks, network) is not True:
                raise ValueError('Network ID is not found.')
            else:
                provisioning_parameters['networks'].append(network)
    elif instance_security_groups is not None:
        provisioning_parameters['ex_security_groups'] = []
        account_securitygroups = security_group_list(connection)
        instance_security_groups = instance_security_groups.split(',')
        for securitygroup in instance_security_groups:
            if find_json(account_securitygroups, securitygroup, 'id') is not True:
                raise ValueError('Security Group ID is not found.')
            else:
                provisioning_parameters['ex_security_groups'].append(find_sg_name(account_securitygroups, securitygroup))
                flat_network = True
    else:
        raise ValueError('Neither Network or Security Group IDs were found. Please specify at '
                         'least one network or security group.')

#    if find_json(instance_keys, instance_ssh_key, 'name') is not True:
#        raise ValueError('SSH Key ID not found.')
#    else:
#        provisioning_parameters['keypair'] = instance_ssh_key

    provisioning_parameters['ex_keyname'] = instance_ssh_key
    print provisioning_parameters
    return connection.create_node(provisioning_parameters)


def main():
    module = AnsibleModule(
        argument_spec=dict(
            access_key=dict(),
            secret_key=dict(),
            api_url=dict(default='http://localhost:8080/client/api'),
            key_name=dict(aliases=['keypair']),
            security_groups=dict(),
            networks=dict(),
            wait=dict(default='yes', choices=['yes', 'no']),
            wait_for=dict(default=30),
            state=dict(default='present', choices=['absent', 'present']),
            user_data=dict(),
            template_id=dict(required=True),
            size_id=dict(aliases=['flavor_id', 'service_offering_id'], required=True),
            zone_id=dict(required=True),
            instance_name=dict(),
            instance_display_name=dict(),
        ),
    )
    connection = initialize_connection(module)
    if module.params['state'] == 'present':
        instance = create_instances(connection, module)
        print "Instance " + instance['id']


# this is magic, see lib/ansible/module_common.py
#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
from ansible.module_utils.basic import *
main()

cloudstack.py-0.0.2

def create_instances(connection, instance):
    '''
    Creates new instances
    '''

    '''
    Run tests to validate deployment information as valid.
    '''
    provisioning_parameters = {}
    cloud_templates = account_images(connection)
    service_offerings = list_sizes(connection)
    cloud_zones = zone_list(connection)
#    instance_keys = account_keypairs(connection)
    instance_template = instance.params['template_id']
    instance_service_offering = instance.params['size_id']
    instance_networks = instance.params['networks']
    instance_security_groups = instance.params['security_groups']
    instance_ssh_key = instance.params['key_name']
    instance_zone_id = instance.params['zone_id']
    flat_network = False

    if find_object(cloud_templates, instance_template) is False:
        raise ValueError(
            'Template ID not found. Please verify the template ID or that you have access to that template.')
    else:
        image = [i for i in cloud_templates if i.id == instance_template][0]

    if find_object(service_offerings, instance_service_offering) is False:
        raise ValueError('Size ID not found. Please verify the Service Offering ID.')
    else:
        size = [s for s in service_offerings if s.id == instance_service_offering][0]

    if find_object(cloud_zones, instance_zone_id) is False:
        raise ValueError('Zone ID not found. Please verify the Zone ID.')
    else:
        location = [z for z in cloud_zones if z.id == instance_zone_id][0]

    if instance_networks is not None:
        provisioning_parameters['networks'] = []
        account_networks = network_list(connection)
        instance_networks = instance_networks.split(',')
        for network in instance_networks:
            if find_object(account_networks, network) is not True:
                raise ValueError('Network ID is not found.')
            else:
                provisioning_parameters['networks'].append(network)
    elif instance_security_groups is not None:
        provisioning_parameters['ex_security_groups'] = []
        account_securitygroups = security_group_list(connection)
        instance_security_groups = instance_security_groups.split(',')
        for securitygroup in instance_security_groups:
            if find_json(account_securitygroups, securitygroup, 'id') is not True:
                raise ValueError('Security Group ID is not found.')
            else:
                provisioning_parameters['ex_security_groups'].append(find_sg_name(account_securitygroups, securitygroup))
                flat_network = True
    else:
        raise ValueError('Neither Network or Security Group IDs were found. Please specify at '
                         'least one network or security group.')

#    if find_json(instance_keys, instance_ssh_key, 'name') is not True:
#        raise ValueError('SSH Key ID not found.')
#    else:
#        provisioning_parameters['keypair'] = instance_ssh_key

    provisioning_parameters['ex_keyname'] = instance_ssh_key
    print provisioning_parameters
    return connection.create_node(image=image, location=location, size=size, provisioning_parameters)

hum weird, what's your exact create_node call ?

connection.create_node(provisioning_parameters)

Where provisioning_parameters is a dict roughly equal to:
"{'ex_keyname': 'jmoody_fifthecho', 'image': <NodeImage: id=85d5a794-c3f4-49fd-afee-19132a850f79, name=Datapipe: CentOS 6 (64-bit), driver=CloudStack ...>, 'location': <NodeLocation: id=5, name=New York Metro, country=Unknown, driver=CloudStack>, 'ex_security_groups': ['default', 'TestSG'], 'size': <NodeSize: id=44, name=kilo-2-20, ram=2048 disk=0 bandwidth=0 price=0 driver=CloudStack ...>}"

right, so I would not do it that way. I would do connection.create_node(image=provisioning_parameters['image'],size=provisioning_parameters['size'],.....

etc....

Problem there is then I need to write tests if it's using security groups, networks, name, displayname, diskoffering, keypair, userdata, projects, or not.

With FOG I can just pass in a hash of appropriately named key-value pairs and it does the heavy lifting to make the right API call, can we get libcloud to behave in a similar way so that I can pass it a dict of key-value pairs and it runs with those?

Tried that with "connection.create_node(image=image, location=location, size=size, provisioning_parameters)" and am getting:
line 329\n return connection.create_node(image=image, location=location, size=size, provisioning_parameters)\nSyntaxError: non-keyword arg after keyword arg\n"