Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Tempest config tool improvements

Things that can be improved in python-tempestconf:

Notes:

  1. --create and --non-admin is not used together as some of the resources requires admin credentials to create tempest resources. Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L268
if args.create and args.non_admin:
        raise Exception("Options '--create' and '--non-admin' cannot be used"
                        " together, since creating" " resources requires"
" admin rights")

Refactor

Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L300

def parse_overrides(overrides):
    """Manual parsing of positional arguments.
    TODO(mkollaro) find a way to do it in argparse
    """
    if len(overrides) % 2 != 0:
        raise Exception("An odd number of override options was found. The"
                        " overrides have to be in 'section.key value' format.")
    i = 0
    new_overrides = []
    while i < len(overrides):
        section_key = overrides[i].split('.')
        value = overrides[i + 1]
        if len(section_key) != 2:
            raise Exception("Missing dot. The option overrides has to come in"
                            " the format 'section.key value', but got '%s'."
                            % (overrides[i] + ' ' + value))
        section, key = section_key
        new_overrides.append((section, key, value))
        i += 2
    return new_overrides

and

Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L277

def parse_values_to_remove(options):
    """Manual parsing of remove arguments.
    :options list of arguments following --remove argument
    :returns dict containing key paths with values to be removed
    EXAMPLE: {'identity.username': [myname],
              'identity-feature-enabled.api_extensions': [http, https]}
    """
    parsed = {}
    for argument in options:
        if len(argument.split('=')) == 2:
            section, values = argument.split('=')
            if len(section.split('.')) != 2:
                raise Exception("Missing dot. The option --remove has to"
                                "come in the format 'section.key=value,"
                                " but got '%s'." % (argument))
            parsed[section] = values.split(',')
        else:
            # missing equal sign, all values in section.key will be deleted
            parsed[argument] = []
return parsed

Both portions needs to be refactored to use same logic for checking the passed section.key values.

  1. Setting values for Identity section while using cloud_config or passing values directly from default ovrrides or while using non-admin accounts are repeated Code link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L323
def set_cloud_config_values(conf, args):
    """Set values from client's cloud config file.
    If the cloud config files was provided, set admin and non-admin credentials
    and uri.
    Note: the values may be later overriden by values specified in CLI.
    :conf TempestConf object
    :args parsed arguments including client config values
    """
    cloud_creds = args.config.get('auth')
    if cloud_creds:
        try:
            if args.non_admin:
                conf.set('identity', 'username', cloud_creds['username'])
                conf.set('identity',
                         'tenant_name',
                         cloud_creds['project_name'])
                conf.set('identity', 'password', cloud_creds['password'])
            else:
                conf.set('identity', 'admin_username', cloud_creds['username'])
                conf.set('identity',
                         'admin_tenant_name',
                         cloud_creds['project_name'])
                conf.set('identity', 'admin_password', cloud_creds['password'])
            conf.set('identity', 'uri', cloud_creds['auth_url'])

        except cfg.NoSuchOptError:
            LOG.warning('Could not load some identity options from cloud config file')

Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L159

if args.non_admin:
        conf.set("auth", "admin_username", "")
        conf.set("auth", "admin_project_name", "")
        conf.set("auth", "admin_password", "")
        # To maintain backward compatibilty
        # Moved to auth
        conf.set("identity", "admin_username", "")
        # To maintain backward compatibility
        # renamed as admin_project_name in auth section
        conf.set("identity", "admin_tenant_name", "")
        # To maintain backward compatibility
        # Moved to auth
        conf.set("identity", "admin_password", "")
        conf.set("auth", "use_dynamic_credentials", "False")

Workarounds

  1. Automatically setting the values from default-overrides or tripleo-deployer-input conf in tempest.conf Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L133
if os.path.isfile(DEFAULTS_FILE):
        LOG.info("Reading defaults from file '%s'", DEFAULTS_FILE)
        conf.read(DEFAULTS_FILE)
    if args.deployer_input and os.path.isfile(args.deployer_input):
        LOG.info("Adding options from deployer-input file '%s'",
                 args.deployer_input)
        deployer_input = ConfigParser.SafeConfigParser()
        deployer_input.read(args.deployer_input)
        for section in deployer_input.sections():
            # There are no deployer input options in DEFAULT
            for (key, value) in deployer_input.items(section):
                conf.set(section, key, value, priority=True)

Setting values: Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L627

def set(self, section, key, value, priority=False):
        """Set value in configuration, similar to `SafeConfigParser.set`
        Creates non-existent sections. Keeps track of options which were
        specified by the user and should not be normally overwritten.
        :param priority: if True, always over-write the value. If False, don't
            over-write an existing value if it was written before with a
            priority (i.e. if it was specified by the user)
        :returns: True if the value was written, False if not (because of
            priority)
        """
        if not self.has_section(section) and section.lower() != "default":
            self.add_section(section)
        if not priority and (section, key) in self.priority_sectionkeys:
            LOG.debug("Option '[%s] %s = %s' was defined by user, NOT"
                      " overwriting into value '%s'", section, key,
                      self.get(section, key), value)
            return False
        if priority:
            self.priority_sectionkeys.add((section, key))
        LOG.debug("Setting [%s] %s = %s", section, key, value)
        ConfigParser.SafeConfigParser.set(self, section, key, value)
    return True

Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L459

def __init__(self, conf, admin):
        self.identity_version = self.get_identity_version(conf)
        username = None
        password = None
        tenant_name = None
        if admin:
            try:
                username = conf.get_defaulted('auth', 'admin_username')
                if username is None:
                    username = conf.get_defaulted('identity', 'admin_username')

                password = conf.get_defaulted('auth', 'admin_password')
                if password is None:
                    password = conf.get_defaulted('identity', 'admin_password')

                tenant_name = conf.get_defaulted('auth',
                                                 'admin_project_name')
                if tenant_name is None:
                    tenant_name = conf.get_defaulted('identity',
                                                     'admin_tenant_name')

            except cfg.NoSuchOptError:
                LOG.warning(
                    'Could not load some identity admin options from %s',
                    DEFAULTS_FILE)
        else:
            try:
                username = conf.get_defaulted('identity', 'username')
                password = conf.get_defaulted('identity', 'password')
                tenant_name = conf.get_defaulted('identity', 'tenant_name')

            except cfg.NoSuchOptError:
                LOG.warning(
                    'Could not load some identity options from %s',
                    DEFAULTS_FILE)

        self.identity_region = conf.get_defaulted('identity', 'region')
        default_params = {
            'disable_ssl_certificate_validation':
                conf.get_defaulted('identity',
                                   'disable_ssl_certificate_validation'),
            'ca_certs': conf.get_defaulted('identity', 'ca_certificates_file')
        }
        compute_params = {
            'service': conf.get_defaulted('compute', 'catalog_type'),
            'region': self.identity_region,
            'endpoint_type': conf.get_defaulted('compute', 'endpoint_type')
        }
        compute_params.update(default_params)

Note: resources related to those values defined in deployer input files is generally not available and it automatically setted which leads to failure So it is better to create the resources and then set it. Questions: * Whether to look whether these resources can be created by admin or demo credentials based on that perform the action.

  1. Setting the value of auth_version and auth_uri by replacing without checking whether it exists/enabled or not. Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L150
    uri = conf.get("identity", "uri")
    api_version = 2
    if "v3" in uri:
        api_version = 3
        conf.set("identity", "auth_version", "v3")
        conf.set("identity", "uri_v3", uri)
    else:
        # TODO(arxcruz) make a check if v3 is enabled
        conf.set("identity", "uri_v3", uri.replace("v2.0", "v3"))
  1. Service API discovery (api_discovery.py) contains lots of hacks.
def no_port_cut_url(self):
     # if there is no port defined, cut the url from version to the end
     u = urllib3.util.parse_url(self.service_url)
     url = self.service_url
     if u.port is None:
         found = re.findall(r'v\d', url)
         if len(found) > 0:
             index = url.index(found[0])
             url = self.service_url[:index]
     return (url, u.port is not None)

And

def get_identity_v3_extensions(keystone_v3_url):
 """Returns discovered identity v3 extensions
 As keystone V3 uses a JSON Home to store the extensions,
 this method is kept  here just for the sake of functionality, but it
 implements a different discovery method.
 :param keystone_v3_url: Keystone V3 auth url
 :return: A list with the discovered extensions
 """
 try:
     r = requests.get(keystone_v3_url,
                      verify=False,
                      headers={'Accept': 'application/json-home'})
 except requests.exceptions.RequestException as re:
     LOG.error("Request on service '%s' with url '%s' failed",
               'identity', keystone_v3_url)
     raise re
 ext_h = 'http://docs.openstack.org/api/openstack-identity/3/ext/'
 res = [x for x in json.loads(r.content)['resources'].keys()]
 ext = [ex for ex in res if 'ext' in ex]
  return list(set([str(e).replace(ext_h, '').split('/')[0] for e in ext]))
  1. Based on api_discovery, we discovers services available do a check whether it is in that list or not and sett the values. Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L861
def check_ceilometer_service(client, conf, services):
    services = client.list_services(**{'type': 'metering'})
    if services and len(services['services']):
        metering = services['services'][0]
        if 'ceilometer' in metering['name'] and metering['enabled']:
            conf.set('service_available', 'ceilometer', 'True')


def check_volume_backup_service(client, conf, services):
    """Verify if the cinder backup service is enabled"""
    if 'volumev3' not in services:
        LOG.info("No volume service found, skipping backup service check")
        return
    params = {'binary': 'cinder-backup'}
    backup_service = client.list_services(**params)

    if backup_service:
        # We only set backup to false if the service isn't running otherwise we
        # keep the default value
        service = backup_service['services']
        if not service or service[0]['state'] == 'down':
conf.set('volume-feature-enabled', 'backup', 'False')
  1. Workaround for configuring horizon by parsing url Code URL: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L988
def configure_horizon(conf):
    """Derive the horizon URIs from the identity's URI."""
    uri = conf.get('identity', 'uri')
    u = urllib2.urlparse.urlparse(uri)
    base = '%s://%s%s' % (u.scheme, u.netloc.replace(
        ':' + str(u.port), ''), '/dashboard')
    assert base.startswith('http:') or base.startswith('https:')
    has_horizon = True
    try:
        urllib2.urlopen(base)
    except urllib2.URLError:
        has_horizon = False
    conf.set('service_available', 'horizon', str(has_horizon))
    conf.set('dashboard', 'dashboard_url', base + '/')
conf.set('dashboard', 'login_url', base + '/auth/login/')

Improvements

  1. Setting the test_accounts file path Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L176
conf.set("auth", "test_accounts_file", "etc/accounts.yaml")
  1. ClientManager(conf, not args.non_admin) sets lots of values to tempest.conf for different services it should be docupled based on services. Code Link: https://github.com/openstack/python-tempestconf/blob/master/config_tempest/config_tempest.py#L459 For example on this step client = ClientManager(conf, not args.non_admin)

    def __init__(self, conf, admin):
        self.identity_version = self.get_identity_version(conf)
        username = None
        password = None
        tenant_name = None
        if admin:
            try:
                username = conf.get_defaulted('auth', 'admin_username')
                if username is None:
                    username = conf.get_defaulted('identity', 'admin_username')

                password = conf.get_defaulted('auth', 'admin_password')
                if password is None:
                    password = conf.get_defaulted('identity', 'admin_password')

                tenant_name = conf.get_defaulted('auth',
                                                 'admin_project_name')
                if tenant_name is None:
                    tenant_name = conf.get_defaulted('identity',
                                                     'admin_tenant_name')

            except cfg.NoSuchOptError:
                LOG.warning(
                    'Could not load some identity admin options from %s',
                    DEFAULTS_FILE)
        else:
            try:
                username = conf.get_defaulted('identity', 'username')
                password = conf.get_defaulted('identity', 'password')
                tenant_name = conf.get_defaulted('identity', 'tenant_name')

            except cfg.NoSuchOptError:
                LOG.warning(
                    'Could not load some identity options from %s',
                    DEFAULTS_FILE)

        self.identity_region = conf.get_defaulted('identity', 'region')
        default_params = {
            'disable_ssl_certificate_validation':
                conf.get_defaulted('identity',
                                   'disable_ssl_certificate_validation'),
            'ca_certs': conf.get_defaulted('identity', 'ca_certificates_file')
        }
        compute_params = {
            'service': conf.get_defaulted('compute', 'catalog_type'),
            'region': self.identity_region,
            'endpoint_type': conf.get_defaulted('compute', 'endpoint_type')
        }
        compute_params.update(default_params)

        if self.identity_version == "v2":
            _creds = self.get_credentials(conf, username, tenant_name,
                                          password)
        else:
            _creds = self.get_credentials(
                conf, username, tenant_name, password,
                identity_version=self.identity_version)

        _auth = self.get_auth_provider(conf, _creds)
        self.auth_provider = _auth

        if "v2.0" in conf.get("identity", "uri"):
            self.identity = identity_client.IdentityClient(
                _auth, conf.get_defaulted('identity', 'catalog_type'),
                self.identity_region, endpoint_type='publicURL',
                **default_params)
        else:
            self.identity = identity_v3_client.IdentityClient(
                _auth, conf.get_defaulted('identity', 'catalog_type'),
                self.identity_region, endpoint_type='publicURL',
                **default_params)

        self.tenants = ProjectsClient(
            _auth,
            conf.get_defaulted('identity', 'catalog_type'),
            self.identity_region,
            'publicURL',
            self.identity_version,
            **default_params)

        self.set_roles_client(
            auth=_auth,
            conf=conf,
            endpoint_type='publicURL',
            default_params=default_params)

        self.set_users_client(
            auth=_auth,
            conf=conf,
            endpoint_type='publicURL',
            default_params=default_params)

        self.images = images_client.ImagesClient(
            _auth,
            conf.get_defaulted('image', 'catalog_type'),
            self.identity_region,
            **default_params)

        self.servers = servers_client.ServersClient(_auth,
                                                    **compute_params)
        self.flavors = flavors_client.FlavorsClient(_auth,
                                                    **compute_params)

        self.networks = None

        self.service_client = s_client.ServicesClient(
            _auth,
            conf.get_defaulted('identity', 'catalog_type'),
            self.identity_region,
            **default_params)

        self.volume_service = services_client.ServicesClient(
            _auth,
            conf.get_defaulted('volume', 'catalog_type'),
            self.identity_region,
            **default_params)

        def create_nova_network_client():
            if self.networks is None:
                self.networks = nova_net_client.NetworksClient(
                    _auth, **compute_params)
            return self.networks

        def create_neutron_client():
            if self.networks is None:
                self.networks = networks_client.NetworksClient(
                    _auth,
                    conf.get_defaulted('network', 'catalog_type'),
                    self.identity_region,
                    endpoint_type=conf.get_defaulted('network',
                                                     'endpoint_type'),
                    **default_params)
            return self.networks

        self.get_nova_net_client = create_nova_network_client
        self.get_neutron_client = create_neutron_client

        # Set admin tenant id needed for keystone v3 tests.
        if admin:
            tenant_id = self.tenants.get_project_by_name(tenant_name)['id']
            conf.set('identity', 'admin_tenant_id', tenant_id)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment