validate_version_added_fields_in_config.py

validate_version_added_fields_in_config.py

import functools
import sys
from pathlib import Path
from typing import List
from pprint import pprint
import requests
import semver
import yaml


def get_all_versions() -> List[str]:
    r = requests.get('https://pypi.org/pypi/apache-airflow/json')
    r.raise_for_status()
    all_version = r.json()['releases'].keys()
    released_versions = [d for d in all_version if not (('rc' in d) or ('b' in d))]
    return released_versions


@functools.lru_cache()
def get_all_config_options_for_version(version: str):
    r = requests.get(
        f'https://raw.githubusercontent.com/apache/airflow/{version}/airflow/config_templates/config.yml'
    )
    r.raise_for_status()
    config_sections = yaml.safe_load(r.text)
    config_options = {
        (
            config_section['name'],
            config_option['name'],
        )
        for config_section in config_sections
        for config_option in config_section['options']
    }
    return config_options


def get_all_config_options_locally():
    config_sections = yaml.safe_load(Path("airflow/config_templates/config.yml").read_text())
    config_options = {
        (config_section['name'], config_option['name'], config_option['version_added'])
        for config_section in config_sections
        for config_option in config_section['options']
    }
    return config_options


# 1. Prepare versions to checks
airflow_version = get_all_versions()
airflow_version = sorted(airflow_version, key=semver.VersionInfo.parse)
airflow_1_10_versions = [d for d in airflow_version if d.startswith("1.10")]
# Skip versions without config.yml file
airflow_1_10_versions = airflow_1_10_versions[8:]
airflow_2_versions = [d for d in airflow_version if d.startswith("2.")]
to_check_versions = [*airflow_1_10_versions, *airflow_2_versions]

# 2. Compute expected options set with version
all_computed_options = set()
for prev_version, curr_version in zip(to_check_versions[:-1], to_check_versions[1:]):
    print("Processing version: ", curr_version)
    options_1 = get_all_config_options_for_version(prev_version)
    options_2 = get_all_config_options_for_version(curr_version)
    new_options = options_2 - options_1
    all_computed_options.update(
        {(section_name, option_name, curr_version) for section_name, option_name in new_options}
    )

# 3. Read local options set
all_local_options = get_all_config_options_locally()

# 4. Hide options that do not exist in the local configuration file. They are probably deprecated.
all_local_options_plain = {
    (section_name, option_name) for section_name, option_name, version_added in all_local_options
}
all_computed_options = {
    (section_name, option_name, version_added)
    for section_name, option_name, version_added in all_computed_options
    if (section_name, option_name) in all_local_options_plain
}

# 5. Compute difference between expected and current set
all_local_options_with_version_added = {
    (section_name, option_name, version_added)
    for section_name, option_name, version_added in all_local_options
    if version_added
}
diff_options = all_computed_options - all_local_options_with_version_added

if diff_options:
    pprint(diff_options)
    sys.exit(1)
else:
    print("No changes required")