remove redundant sql files

sql-deduplication.py

PATH_TO_PGADMIN = "../pgadmin4"

import subprocess
import os, re


def remove_from_higher_directory(higher_directory, lower_directory):
    #figure out which templates are unchanged between directories
    diff_output, err = subprocess.Popen(["diff",
                                         lower_directory,
                                         higher_directory],
                                        stdout=subprocess.PIPE).communicate()

    grep_output, err = subprocess.Popen(["grep", "^diff"], stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate(
        diff_output)

    diff_lines = grep_output.split("\n")

    #Don't remove files that don't exist in lower versions
    files_to_remove = map(os.path.basename, os.listdir(lower_directory))

    for line in diff_lines:
        if (len(line) <= 1):
            continue
        _, lower_file, higer_file = line.split(" ")

        print ('not removing ' + os.path.basename(higer_file))
        files_to_remove.remove(os.path.basename(higer_file))

    for file in files_to_remove:
        full_path = higher_directory + '/' + file
        print(full_path)
        if os.path.exists(full_path):
            os.remove(full_path)


sql_directories_string, err = subprocess.Popen(["find", PATH_TO_PGADMIN, "-path", "*/sql/*", "-type", "d"],
                                               stdout=subprocess.PIPE).communicate()
sql_directories = sql_directories_string.split("\n")
directory_map = dict()

for sql_directory in sql_directories:
    try:
        # find all the directories that can be handled by our template loader
        feature_directory, version_suffix = re.match("(.*)/(9.*plus)", sql_directory).groups()
    except AttributeError:
        continue

    if feature_directory not in directory_map:
        directory_map[feature_directory] = list()
    directory_map[feature_directory].append(version_suffix)

print(directory_map)

for key, value in directory_map.iteritems():
    while len(value) >= 2:
        # only remove from the higher directory, always leave at least one copy of each template
        higher_directory = value.pop()
        remove_from_higher_directory(key + "/" + higher_directory, key + "/" + value[-1])