Given the evergreen.yml file, create a new file which moves all MMAPv1 tasks to a new set of MMAPv1-only buildvariants.

deprecate_mmap_testing.py

'''Given the evergreen.yml file, this creates a new file which moves all MMAPv1 tasks
to a new set of MMAPv1-only buildvariants.
'''
# /usr/bin/python3

from ruamel.yaml import YAML


def main():
    '''The thing that does the work.'''
    yaml = YAML()
    with open("/Users/ian/code/mongodb/mongo/etc/evergreen.yml", 'r') as stream:
        data = yaml.load(stream)

        add_new_buildvariants(data['buildvariants'])

        for task in data['tasks']:
            resmoke_args = list(gen_dict_extract('resmoke_args', task))
            if resmoke_args:
                if '--storageEngine=mmapv1' in resmoke_args[0]:
                    # Add the task using MMAPv1 to each of the three MMAP-specific buildvariants.
                    for variant in data['buildvariants'][-3:]:
                        tasks = variant['tasks']
                        tasks.append({'name': task['name']})
                    # Remove the task using MMAPv1 from all other buildvariants.
                    for variant in data['buildvariants'][:-3]:
                        tasks = variant['tasks']
                        print('\n')
                        print(tasks)
                        tasks[:] = [d for d in tasks if d.get(
                            'name') != task['name']]

        with open('/Users/ian/Desktop/out.yml', 'w') as output_file:
            yaml.dump(data, output_file)


def add_new_buildvariants(bv_list):
    '''Append base information for new MMAP buildvariants.'''
    rhel_bv = {
        "name": "enterprise-rhel-62-64-bit-mmapv1",
        "display_name": "Enterprise RHEL 6.2 MMAPv1",
        "modules": ["enterprise"],
        "run_on": ["rhel62-small"],
        "batchtime": "1440",  # 1 day
        "expansions": {
            "gorootvars": 'GOROOT=/opt/go PATH="/opt/go/bin:$PATH"',
            "tooltags": "-tags 'ssl sasl'",
            "rlp_environment": "MONGOD_UNITTEST_RLP_LANGUAGE_TEST_BTROOT=/opt/basis",
            "compile_flags": '--ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) '\
                             '--release --variables-files=etc/scons/mongodbtoolchain_gcc.vars '\
                             'CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include" '\
                             '--use-basis-tech-rosette-linguistics-platform=on',
            "multiversion_platform_arch": "rhel62",
            "multiversion_edition": "targeted",
            "num_jobs_available": "$(grep -c ^processor /proc/cpuinfo)",
            "has_packages": "true",  # TODO maybe anything packaging related can be removed?
            "packager_script": "packager-enterprise.py",
            "packager_arch": "x86_64",
            "packager_distro": "rhel62",
            "repo_edition": "enterprise",
            "use_scons_cache": "true",
            "build_mongoreplay": "true"  # TODO maybe mongoreplay lines can be deleted?
        },
        "tasks": [{"name": "compile"}]}

    win_bv = {
        "name": "enterprise-windows-64-2k8-mmapv1",
        "display_name": "Enterprise Windows 2008R2 MMAPv1",
        "modules": ["enterprise"],
        "run_on": ["windows-64-vs2015-small"],
        "batchtime": "1440",  # 1 day
        "expansions": {
            "platform_decompress": "unzip",
            "tooltags": "-tags 'ssl sasl'",
            "exe": ".exe",
            "gorootvars": 'PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"'\
                          ' CGO_CFLAGS="-D_WIN32_WINNT=0x0601 -DNTDDI_VERSION=0x06010000"',
            "msi_target": "msi",
            "content_type": "application/zip",
            "compile_flags": '--release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include'\
                             ' c:/sasl/include c:/snmp/include c:/curl/include" '\
                             ' LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib"'\
                             ' -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows'\
                             ' --win-version-min=ws08r2',
            # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors
            # due to spawning a large number of linker processes.
            "num_scons_compile_all_jobs_available": "$(( $(grep -c ^processor /proc/cpuinfo) / 4 ))",
            "python": "python",
            "num_jobs_available": "$(grep -c ^processor /proc/cpuinfo)",
            "ext": "zip",
            "use_scons_cache": "true"
        },
        "tasks": [{"name": "compile"}]}

    mac_bv = {
        "name": "enterprise-osx-1010-mmapv1",
        "display_name": "Enterprise OS X 10.10 MMAPv1",
        "modules": ["enterprise"],
        "run_on": ["macos-1012"],
        "batchtime": "1440",  # 1 day
        "expansions": {
            "tooltags": "-tags 'ssl sasl'",
            "gorootvars": "CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include "\
                          "CGO_CFLAGS=-mmacosx-version-min=10.10 "\
                          "CGO_LDFLAGS=-mmacosx-version-min=10.10",
            "compile_env": "DEVELOPER_DIR=/Applications/Xcode8.3.app",
            "compile_flags": '--ssl -j$(sysctl -n hw.logicalcpu) --release --libc++ '\
                             'CCFLAGS="-mmacosx-version-min=10.10" '\
                             'LINKFLAGS="-mmacosx-version-min=10.10" '\
                             'CPPPATH=/opt/mongodbtoolchain/v2/include',
            "num_jobs_available": "1",
            "build_mongoreplay": "true"
        },
        "tasks": [{"name": "compile"}]}

    bv_list.extend([rhel_bv, win_bv, mac_bv])


def gen_dict_extract(key, iterable):
    '''Given a key and an iterable, return the value for that key regardless of where that key is
    in the iterable including embedded lists, dictionaries, etc.'''
    if hasattr(iterable, 'items'):
        for k, v in iterable.items():
            if k == key:
                yield v
            if isinstance(v, dict):
                for result in gen_dict_extract(key, v):
                    yield result
            elif isinstance(v, list):
                for d in v:
                    for result in gen_dict_extract(key, d):
                        yield result


if __name__ == "__main__":
    main()