Dynamic Inventory differences between juno and kilo

dynamic_inventory.diff

20d19
< import hashlib
42c41
<     'is_metal',
---
>     'properties',
43a43
>     'physical_host_group',
45a46
>     'container_networks',
55c56
<         description='Rackspace Openstack, Inventory Generator',
---
>         description='OpenStack Inventory Generator',
81a83,84
>     except AttributeError:
>         return None
85c88
<             ' range in your rpc_user_config.yml.' % name
---
>             ' range in your openstack_user_config.yml.' % name
120c123
<                            is_metal, assignment):
---
>                            properties, assignment):
135c138
<     :param is_metal: ``bol``  If true, a container entry will not be built
---
>     :param properties: ``dict``  Container properties
138a142,145
>     is_metal = False
>     if properties:
>         is_metal = properties.get('is_metal', False)
> 
179c186
<                 'is_metal': is_metal,
---
>                 'properties': properties,
183a191
>                 'physical_host_group': physical_host_type,
219a228,232
>         is_metal = False
>         properties = hdata.get('properties')
>         if properties:
>             is_metal = properties.get('is_metal', False)
> 
233c246
<                 elif hdata.get('is_metal') is True:
---
>                 elif is_metal is True:
239c252
<                 elif hdata.get('is_metal') is True:
---
>                 elif is_metal is True:
262c275
<                         if limit is None or limit in container:
---
>                         if limit is None or (component and limit in component):
267c280
<                          inventory, is_metal):
---
>                          inventory, properties):
275c288
<     :param is_metal: ``bol``  If true, a container entry will not be built
---
>     :param properties: ``dict``  Dict of container properties
305,307c318,320
<                 ' adjust the host entries in your "rpc_user_config.yml" to use'
<                 ' a short hostname. The recommended hostname length is < 20'
<                 ' characters long.' % (host_type, container_name)
---
>                 ' adjust the host entries in your "openstack_user_config.yml"'
>                 ' to use a short hostname. The recommended hostname length is'
>                 ' < 20 characters long.' % (host_type, container_name)
328c341
<             is_metal,
---
>             properties,
350a364
>     hvs = inventory['_meta']['hostvars']
363c377
<                 inventory['_meta']['hostvars'][_key].update({
---
>                 hvs[_key].update({
366a381
>                     'physical_host_group': key
368a384,390
>                 # If the entry is missing the properties key add it.
>                 properties = hvs[_key].get('properties')
>                 if not properties or not isinstance(properties, dict):
>                     hvs[_key]['properties'] = dict()
> 
>                 hvs[_key]['properties'].update({'is_metal': is_metal})
> 
371c393
<                         inventory['_meta']['hostvars'][_key][_k] = _v
---
>                         hvs[_key][_k] = _v
422c444,461
< def _add_additional_networks(key, inventory, ip_q, k_name, netmask):
---
> def _load_optional_q(config, cidr_name):
>     """Load optional queue with ip addresses.
> 
>     :param config: ``dict``  User defined information
>     :param cidr_name: ``str``  Name of the cidr name
>     """
>     cidr = config.get(cidr_name)
>     ip_q = None
>     if cidr is not None:
>         ip_q = Queue.Queue()
>         _load_ip_q(cidr=cidr, ip_q=ip_q)
>     return ip_q
> 
> 
> def _add_additional_networks(key, inventory, ip_q, q_name, netmask, interface,
>                              bridge, net_type, net_mtu, user_config,
>                              is_ssh_address, is_container_address,
>                              static_routes):
428,432c467,511
<     :param key: ``str`` Component key name
<     :param inventory: ``dict``  Living dictionary of inventory
<     :param ip_q: ``object`` build queue of IP addresses
<     :param k_name: ``str`` key to use in host vars for storage
<     """
---
>     :param key: ``str`` Component key name.
>     :param inventory: ``dict``  Living dictionary of inventory.
>     :param ip_q: ``object`` build queue of IP addresses.
>     :param q_name: ``str`` key to use in host vars for storage.
>     :param netmask: ``str`` netmask to use.
>     :param interface: ``str`` interface name to set for the network.
>     :param user_config: ``dict`` user defined configuration details.
>     :param is_ssh_address: ``bol`` set this address as ansible_ssh_host.
>     :param is_container_address: ``bol`` set this address to container_address.
>     :param static_routes: ``list`` List containing static route dicts.
>     """
>     def network_entry():
>         """Return a network entry for a container."""
> 
>         # TODO(cloudnull) After a few releases this conditional should be
>         # simplified. The container address checking that is ssh address
>         # is only being done to support old inventory.
> 
>         if is_metal:
>             _network = dict()
>         else:
>             _network = {'interface': interface}
> 
>         if bridge:
>             _network['bridge'] = bridge
> 
>         if net_type:
>             _network['type'] = net_type
> 
>         if net_mtu:
>             _network['mtu'] = net_mtu
> 
>         return _network
> 
>     def return_netmask():
>         """Return the netmask for a container."""
>         # TODO(cloudnull) After a few releases this conditional should be
>         # simplified. The container address checking that is ssh address
>         # is only being done to support old inventory.
>         _old_netmask = container.get(old_netmask)
>         if _old_netmask:
>             return container.pop(old_netmask)
>         elif netmask:
>             return netmask
> 
434d512
<     addr_name = '%s_address' % k_name
439c517,531
<             _add_additional_networks(group, inventory, ip_q, k_name, netmask)
---
>             _add_additional_networks(
>                 group,
>                 inventory,
>                 ip_q,
>                 q_name,
>                 netmask,
>                 interface,
>                 bridge,
>                 net_type,
>                 net_mtu,
>                 user_config,
>                 is_ssh_address,
>                 is_container_address,
>                 static_routes
>             )
441,450c533,539
<     if 'hosts' in lookup and lookup['hosts']:
<         for chost in lookup['hosts']:
<             container = base_hosts[chost]
<             if not container.get(addr_name):
<                 if ip_q is None:
<                     container[addr_name] = None
<                 else:
<                     container[addr_name] = get_ip_address(
<                         name=k_name, ip_q=ip_q
<                     )
---
>     # Make sure the lookup object has a value.
>     if lookup:
>         hosts = lookup.get('hosts')
>         if not hosts:
>             return
>     else:
>         return
452,454c541,546
<             netmask_name = '%s_netmask' % k_name
<             if netmask_name not in container:
<                 container[netmask_name] = netmask
---
>     # TODO(cloudnull) after a few releases this should be removed.
>     if q_name:
>         old_address = '%s_address' % q_name
>     else:
>         old_address = '%s_address' % interface
>     old_netmask = '%s_netmask' % q_name
455a548,549
>     for container_host in hosts:
>         container = base_hosts[container_host]
457,458c551,553
< def _load_optional_q(config, cidr_name):
<     """Load optional queue with ip addresses.
---
>         # TODO(cloudnull) after a few releases this should be removed.
>         # This removes the old container network value that now serves purpose.
>         container.pop('container_network', None)
460,468c555,631
<     :param config: ``dict``  User defined information
<     :param cidr_name: ``str``  Name of the cidr name
<     """
<     cidr = config.get(cidr_name)
<     ip_q = None
<     if cidr is not None:
<         ip_q = Queue.Queue()
<         _load_ip_q(cidr=cidr, ip_q=ip_q)
<     return ip_q
---
>         if 'container_networks' in container:
>             networks = container['container_networks']
>         else:
>             networks = container['container_networks'] = dict()
> 
>         is_metal = False
>         properties = container.get('properties')
>         if properties:
>             is_metal = properties.get('is_metal', False)
> 
>         # This should convert found addresses based on q_name + "_address"
>         #  and then build the network if its not found.
>         if not is_metal and old_address not in networks:
>             network = networks[old_address] = network_entry()
>             if old_address in container and container[old_address]:
>                 network['address'] = container.pop(old_address)
>             elif not is_metal:
>                 address = get_ip_address(name=q_name, ip_q=ip_q)
>                 if address:
>                     network['address'] = address
> 
>             network['netmask'] = return_netmask()
>         elif is_metal:
>             network = networks[old_address] = network_entry()
>             network['netmask'] = return_netmask()
>             # TODO(cloudnull) After a few releases this conditional should be
>             # simplified. The container address checking that is ssh address
>             # is only being done to support old inventory.
>             if old_address in container and container[old_address]:
>                 network['address'] = container.pop(old_address)
>             else:
>                 if is_ssh_address or is_container_address:
>                     # Container physical host group
>                     cphg = container.get('physical_host_group')
> 
>                     # user_config data from the container physical host group
>                     phg = user_config[cphg][container_host]
>                     network['address'] = phg['ip']
> 
>         if is_ssh_address is True:
>             container['ansible_ssh_host'] = networks[old_address]['address']
> 
>         if is_container_address is True:
>             container['container_address'] = networks[old_address]['address']
> 
>         if static_routes:
>             # NOTE: networks[old_address]['static_routes'] will get
>             #       regenerated on each run
>             networks[old_address]['static_routes'] = []
>             for s in static_routes:
>                 # only add static routes if they are specified correctly
>                 if 'cidr' in s and 'gateway' in s:
>                     networks[old_address]['static_routes'].append(s)
> 
> 
> def _net_address_search(provider_networks, main_netowrk, key):
>     """Set the key netwokr type to the main network if not specified.
> 
>     :param provider_networks: ``list`` Network list of ``dict``s
>     :param main_netowrk: ``str`` The name of the main network bridge.
>     :param key: ``str`` The name of the key to set true.
>     """
>     for pn in provider_networks:
>         # p_net are the provider_network values
>         p_net = pn.get('network')
>         if p_net:
>             # Check for the key
>             if p_net.get(key):
>                 break
>     else:
>         for pn in provider_networks:
>             p_net = pn.get('network')
>             if p_net:
>                 if p_net.get('container_bridge') == main_netowrk:
>                     p_net[key] = True
> 
>     return provider_networks
487c650
<                     value.get('is_metal', False)
---
>                     value.get('properties')
502,536c665,707
<         mgmt_bridge = overrides['management_bridge']
<         mgmt_dict = {}
<         if cidr_networks:
<             for pn in overrides['provider_networks']:
<                 network = pn['network']
<                 if 'ip_from_q' in network and 'group_binds' in network:
<                     q_name = network['ip_from_q']
<                     for group in network['group_binds']:
<                         _add_additional_networks(
<                             key=group,
<                             inventory=inventory,
<                             ip_q=provider_queues[q_name],
<                             k_name=q_name,
<                             netmask=provider_queues['%s_netmask' % q_name]
<                         )
< 
<                 if mgmt_bridge == network['container_bridge']:
<                     nci = network['container_interface']
<                     ncb = network['container_bridge']
<                     ncn = network.get('ip_from_q')
<                     mgmt_dict['container_interface'] = nci
<                     mgmt_dict['container_bridge'] = ncb
<                     if ncn:
<                         cidr_net = netaddr.IPNetwork(cidr_networks.get(ncn))
<                         mgmt_dict['container_netmask'] = str(cidr_net.netmask)
< 
<         for host, hostvars in inventory['_meta']['hostvars'].iteritems():
<             base_hosts = inventory['_meta']['hostvars'][host]
<             if 'container_network' not in base_hosts:
<                 base_hosts['container_network'] = mgmt_dict
< 
<             for _key, _value in hostvars.iteritems():
<                 if _key == 'ansible_ssh_host' and _value is None:
<                     ca = base_hosts['container_address']
<                     base_hosts['ansible_ssh_host'] = ca
---
>         # iterate over a list of provider_networks, var=pn
>         pns = overrides.get('provider_networks', list())
>         pns = _net_address_search(
>             provider_networks=pns,
>             main_netowrk=config['global_overrides']['management_bridge'],
>             key='is_ssh_address'
>         )
> 
>         pns = _net_address_search(
>             provider_networks=pns,
>             main_netowrk=config['global_overrides']['management_bridge'],
>             key='is_container_address'
>         )
> 
>         for pn in pns:
>             # p_net are the provider_network values
>             p_net = pn.get('network')
>             if not p_net:
>                 continue
> 
>             q_name = p_net.get('ip_from_q')
>             ip_from_q = provider_queues.get(q_name)
>             if ip_from_q:
>                 netmask = provider_queues['%s_netmask' % q_name]
>             else:
>                 netmask = None
> 
>             for group in p_net.get('group_binds', list()):
>                 _add_additional_networks(
>                     key=group,
>                     inventory=inventory,
>                     ip_q=ip_from_q,
>                     q_name=q_name,
>                     netmask=netmask,
>                     interface=p_net['container_interface'],
>                     bridge=p_net['container_bridge'],
>                     net_type=p_net.get('container_type'),
>                     net_mtu=p_net.get('container_mtu'),
>                     user_config=config,
>                     is_ssh_address=p_net.get('is_ssh_address'),
>                     is_container_address=p_net.get('is_container_address'),
>                     static_routes=p_net.get('static_routes')
>                 )
544,546c715,716
<       /etc/rpc_deploy/
<       $HOME/rpc_deploy/
<       $(pwd)/rpc_deploy/
---
>       /etc/openstack_deploy/
>       $(pwd)/openstack_deploy/
551d720
< 
553,554c722
<         os.path.join('/etc', 'rpc_deploy'),
<         os.path.join(os.environ.get('HOME'), 'rpc_deploy')
---
>         os.path.join('/etc', 'openstack_deploy'),
593,594c761,765
<         if 'ansible_ssh_host' in host_entry:
<             append_if(array=USED_IPS, item=host_entry['ansible_ssh_host'])
---
>         networks = host_entry.get('container_networks', dict())
>         for network_entry in networks.values():
>             address = network_entry.get('address')
>             if address:
>                 append_if(array=USED_IPS, item=address)
596,598d766
<         for key, value in host_entry.iteritems():
<             if key.endswith('address'):
<                 append_if(array=USED_IPS, item=value)
600,601c768
< 
< def _ensure_inventory_uptodate(inventory):
---
> def _ensure_inventory_uptodate(inventory, container_skel):
615a783,793
>                 if rh == 'container_networks':
>                     value[rh] = {}
> 
>     for key, value in container_skel.iteritems():
>         item = inventory.get(key)
>         hosts = item.get('hosts')
>         if hosts:
>             for host in hosts:
>                 container = inventory['_meta']['hostvars'][host]
>                 if 'properties' in value:
>                     container['properties'] = value['properties']
653,680d830
< def md5_checker(localfile):
<     """Check for different Md5 in CloudFiles vs Local File.
< 
<     If the md5 sum is different, return True else False
< 
<     :param localfile:
<     :return True|False:
<     """
< 
<     def calc_hash():
<         """Read the hash.
< 
<         :return data_hash.read():
<         """
< 
<         return data_hash.read(128 * md5.block_size)
< 
<     if os.path.isfile(localfile) is True:
<         md5 = hashlib.md5()
<         with open(localfile, 'rb') as data_hash:
<             for chk in iter(calc_hash, ''):
<                 md5.update(chk)
< 
<         return md5.hexdigest()
<     else:
<         raise SystemExit('This [ %s ] is not a file.' % localfile)
< 
< 
698c848
<     """Discover new items in a conf.d directory and add the new values.
---
>     """Discover new items in any extra directories and add the new values.
724c874
<     user_config_file = os.path.join(local_path, 'rpc_user_config.yml')
---
>     user_config_file = os.path.join(local_path, 'openstack_user_config.yml')
738c888
<             'No rpc_user_config files are available in either the base'
---
>             'No openstack_user_config files are available in either the base'
743c893
<     environment_file = os.path.join(local_path, 'rpc_environment.yml')
---
>     environment_file = os.path.join(local_path, 'openstack_environment.yml')
745c895
<     # Load existing rpc environment json
---
>     # Load existing openstack environment json
749,759c899,902
<     # Check the version of the environment file
<     env_version = md5_checker(localfile=environment_file)
<     version = user_defined_config.get('environment_version')
<     if env_version != version:
<         raise SystemExit(
<             'The MD5 sum of the environment file does not match the expected'
<             ' value. To ensure that you are using the proper environment'
<             ' please repull the correct environment file from the upstream'
<             ' repository. Found MD5: [ %s ] expected MD5 [ %s ]'
<             % (env_version, version)
<         )
---
>     # Load anything in an env.d directory if found
>     env_plugins = os.path.join(local_path, 'env.d')
>     if os.path.isdir(env_plugins):
>         _extra_config(user_defined_config=environment, base_dir=env_plugins)
762c905,907
<     dynamic_inventory_file = os.path.join(local_path, 'rpc_inventory.json')
---
>     dynamic_inventory_file = os.path.join(
>         local_path, 'openstack_inventory.json'
>     )
770c915
<             'backup_rpc_inventory.tar'
---
>             'backup_openstack_inventory.tar'
783,785c928,929
<     if 'container' in user_defined_config.get('cidr_networks', list()):
<         user_cidr = user_defined_config['cidr_networks']['container']
<     else:
---
>     cidr_networks = user_defined_config.get('cidr_networks')
>     if not cidr_networks:
787a932,938
>     if 'container' in cidr_networks:
>         user_cidr = cidr_networks['container']
>     elif 'management' in cidr_networks:
>         user_cidr = cidr_networks['management']
>     else:
>         raise SystemExit('No container or management network specified')
> 
800c951,952
<         environment.get('component_skel'), dynamic_inventory
---
>         environment.get('component_skel'),
>         dynamic_inventory
809c961,964
<     _ensure_inventory_uptodate(inventory=dynamic_inventory)
---
>     _ensure_inventory_uptodate(
>         inventory=dynamic_inventory,
>         container_skel=environment.get('container_skel'),
>     )
827c982,983
<     with open(os.path.join(local_path, 'rpc_hostnames_ips.yml'), 'wb') as f:
---
>     hostnames_ip_file = os.path.join(local_path, 'openstack_hostnames_ips.yml')
>     with open(hostnames_ip_file, 'wb') as f: