Change Spark version in each submodule

change_spark_version.py

"""
Small script to change Apache Spark version in all the modules.
It has been tested against Spark 3.0.1
We used it to rebuild Spark against different versions of the dependencies
"""
import os


def main():
    old_version = "3.0.1"
    new_version = "3.0.1-my-fancy-version-SNAPSHOT"

    # replace_pom_version_in_place('pom-test.xml', old_version, new_version)

    for root, dirs, files in os.walk(".", topdown=False):
        for name in files:
            if name == "pom.xml":
                replace_pom_version_in_place(os.path.join(root, name), old_version, new_version)

    replace_version_in_place(os.path.join('R', 'pkg', 'DESCRIPTION'), f"Version: {old_version}", f"Version: {new_version}")
    replace_version_in_place(os.path.join('docs', '_config.yml'), f"SPARK_VERSION: {old_version}", f"SPARK_VERSION: {new_version}")
    replace_version_in_place(os.path.join('R', 'pkg', 'DESCRIPTION'), f"SPARK_VERSION_SHORT: {old_version}", f"SPARK_VERSION_SHORT: {new_version}")
    replace_version_in_place(os.path.join('python', 'pyspark', 'version.py'), f'__version__ = "{old_version}"', f'__version__ = "{new_version}"')




def replace_version_in_place(file, old_version_identifier, new_version_identifier):
    with open(file, 'r') as f:
        lines = []
        for line in f.readlines():
            if old_version_identifier in line:
                line = line.replace(old_version_identifier, new_version_identifier)
            lines.append(line)

    with open(file, 'w') as f:
        f.write(''.join(lines))


def replace_pom_version_in_place(file, old_version, new_version):
    old_version_with_tags = add_version_tag(old_version)
    new_version_with_tags = add_version_tag(new_version)
    replace_version_in_place(file, old_version_with_tags, new_version_with_tags)

def add_version_tag(version):
    return f"<version>{version}</version>"


if __name__ == "__main__":
    main()