gistfile1.txt

diff mono_deduped_commentless.txt mono_deduped_commentless_vars_deduped.txt
1,7d0
< shopt -s extglob
< export PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin/"
< FIRST_BOOT_FILE="/config/firstboot"
< LOGGER_TAG="openstack-init"
< LOGGER_CMD="logger -t $LOGGER_TAG"
< STATUS_CHECK_RETRIES=20
< STATUS_CHECK_INTERVAL=10
9,15d1
< OS_CONFIG_DRIVE_MOUNT_POINT="/config/OPENSTACK_CONFIG_DRIVE"
< OS_CONFIG_DRIVE_META_DATA_FILE="/openstack/latest/meta_data.json"
< OS_CONFIG_DRIVE_USER_DATA_FILE="/openstack/latest/user_data"
< OS_USER_DATA_LOCAL_FILE="/config/user_data.json"
< OS_USER_DATA_RETRIES=20
< OS_USER_DATA_RETRY_INTERVAL=10
< OS_USER_DATA_RETRY_MAX_TIME=300
18,28d3
< OS_USER_DATA_TMP_FILE="/tmp/openstack-user-data.json"
< OS_META_DATA_TMP_FILE="/tmp/openstack-meta-data.json"
< OS_USER_DATA_CLEANUP=true
< OS_META_DATA_CLEANUP=true
< OS_SSH_KEY_INJECT_ENABLED=true
< OS_SSH_KEY_RETRIES=5
< OS_SSH_KEY_RETRY_INTERVAL=10
< OS_SSH_KEY_RETRY_MAX_TIME=300
< OS_SSH_KEY_PATH="/latest/meta-data/public-keys/0/openssh-key"
< OS_SSH_KEY_TMP_FILE="/tmp/openstack-ssh-key.pub"
< ROOT_AUTHORIZED_KEYS="/root/.ssh/authorized_keys"
34,39d8
< OS_CHANGE_PASSWORDS=true
< BIGIP_LICENSE_FILE="/config/bigip.license"
< BIGIP_LICENSE_RETRIES=5
< BIGIP_LICENSE_RETRY_INTERVAL=5
< BIGIP_PROVISIONING_ENABLED=true
< BIGIP_AUTO_PROVISIONING_ENABLED=true
42,62d10
< OS_MGMT_LEASE_FILE="/var/lib/dhclient/dhclient.leases"
< OS_MGMT_MTU=1400
< OS_DHCP_ENABLED=true
< OS_DHCP_LEASE_FILE="/tmp/openstack-dhcp.leases"
< OS_DHCP_REQ_TIMEOUT=30
< OS_VLAN_PREFIX="openstack-network-"
< OS_VLAN_DESCRIPTION="auto-added by openstack-init"
< OS_VLAN_MTU=1500
< OS_SELFIP_PREFIX="openstack-dhcp-"
< OS_SELFIP_ALLOW_SERVICE="none"
< OS_SELFIP_DESCRIPTION="auto-added by openstack-init"
< OS_DEVICE_SYNC="false"
< OS_DEVICE_FAILOVER="false"
< OS_DEVICE_MIRROR_PRIMARY="false"
< OS_DEVICE_MIRROR_SECONDARY="false"
< LEVEL_REGEX='^(dedicated|minimum|nominal|none)$'
< PW_REGEX='^\$[0-9][A-Za-z]?\$'
< TMM_IF_REGEX='^1\.[0-9]$'
< IP_REGEX='^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
< SELFIP_ALLOW_SERVICE_REGEX='^(all|default|none)$'
< OS_MARKER_STARTUP_COMPLETE="/tmp/openstack_auto_config_completed"

remove_duplicated_variables.py

#! /usr/bin/env python

import re

NONWS = re.compile(r'\S')
VARDEF = re.compile(r'\s*(?P<modifier>readonly |local )?(?P<VARNAME>\w+)=')#(?P<VARNAME>\s+)')
MVARDEF = re.compile(r'^(?P<MVARNAME>\w+)=')#(?P<VARNAME>\s+)')
refactored_lines = open('startup').readlines()

refactored_lines.extend(open('os-functions/openstack-datasource.sh').readlines())
refactored_lines.extend(open('os-functions/openstack-license.sh').readlines())
refactored_lines.extend(open('os-functions/openstack-network.sh').readlines())
refactored_lines.extend(open('os-functions/openstack-password.sh').readlines())
refactored_lines.extend(open('os-functions/openstack-ssh.sh').readlines())

VARS = {}
for line in refactored_lines:
    mo = re.search(VARDEF, line)
    if mo and mo.group('VARNAME'):
        VARS[mo.group('VARNAME').upper()] = line

from pprint import pprint as pp

#pp(VARS)
#print(len(VARS))
comment_less_lines = open('mono_deduped_commentless.txt').readlines()

for line in comment_less_lines:
    if line.startswith('function'):
        print(line.rstrip('\n'))
    elif line.startswith('}'):
        print(line.rstrip('\n'))
    elif re.match(r'\s', line):
        print(line.rstrip('\n'))
    else:
        mo = re.search(MVARDEF, line)
        if mo:
            if mo.group('MVARNAME').upper() not in VARS:
                print(line.rstrip('\n'))