HTTPS Spotter

https_spotter.py

#!/usr/local/autopkg/python
# encoding: utf-8

# HTTPS Spotter
# Copyright 2016-2020 Elliot Jordan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""https_spotter.py.

Scans for HTTP URLs in a specified repository of AutoPkg recipes and suggests
HTTPS alternatives wherever possible.
"""

import argparse
import os
import plistlib
import subprocess

# Path to curl.
CURL_PATH = "/usr/bin/curl"


def build_argument_parser():
    """Build and return the argument parser."""

    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        "repo_path", help="Path to search for AutoPkg recipes."
    )
    parser.add_argument(
        "--auto",
        action="store_true",
        default=False,
        help="Automatically apply suggested changes to recipes. Only recommended "
        "for repos you manage or are submitting a pull request to. (Applying "
        "changes to repos added using `autopkg repo-add` may result in failure "
        "to update the repo in the future.)",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        help="Print additional output useful for "
        "troubleshooting. (Can be specified multiple times.)",
    )
    return parser


def check_url(url, verbose):
    """Checks a given HTTPS URL to see whether an HTTPS equivalent is
    available."""

    https_url = url.replace("http:", "https:")

    try:
        cmd = [
            CURL_PATH,
            "--head",
            "--silent",
            "--location",
            "--max-time",
            "5",
            "--max-redirs",
            "10",
            "--url",
            https_url,
        ]
        if verbose > 1:
            print("  Curl command: %s" % " ".join(cmd))
        proc = subprocess.run(cmd, check=False, capture_output=True)
        http_response = [x for x in proc.stdout.splitlines() if x.startswith(b"HTTP/")]
        http_status = int(http_response[0].split()[1])
        if verbose > 2:
            print("  Exit code: %s" % proc.returncode)
            print("  HTTP status: %s" % http_status)
        if proc.returncode == 0:
            # Ignoring HTTP status because 404 with SSL is better than 404 without.
            return https_url
    except Exception as err:
        if verbose > 3:
            print("  %s" % err)

    return False


def process_recipe(recipe_path, args):
    """Processes one recipe."""

    try:
        # Read recipe.
        with open(recipe_path, "rb") as openfile:
            recipe = plistlib.load(openfile)
        if "Process" in recipe:
            processors = [x.get("Processor") for x in recipe["Process"]]
            if "DeprecationWarning" in processors:
                if args.verbose > 0:
                    print("Skipping: %s (contains DeprecationWarning)" % (recipe_path))
                return
            if args.verbose > 0:
                print("Processing: %s" % (recipe_path))

        urls_to_check = []

        # Gather HTTP URLs from Input.
        for key in recipe.get("Input", {}):
            if isinstance(recipe["Input"][key], str) and recipe["Input"][
                key
            ].startswith("http:"):
                urls_to_check.append(recipe["Input"][key])

        # Gather HTTP URLs from Processors.
        for proc in recipe.get("Process", []):
            for _, value in proc.get("Arguments", {}).items():
                if not isinstance(value, str):
                    # Only looking at string arguments, not diving
                    # deeper into lists/dicts.
                    continue
                if value.startswith("http:"):
                    urls_to_check.append(value)

        # Check HTTP URLs for HTTPS equivalents.
        for http_url in urls_to_check:
            https_url = check_url(http_url, args.verbose)
            if not https_url:
                continue
            if args.auto:
                # Read/write as text instead of using plistlib in order
                # to prevent unrelated changes (e.g. whitespace) in diff.
                with open(recipe_path, "r") as openfile:
                    recipe_contents = openfile.read()
                with open(recipe_path, "w") as openfile:
                    openfile.write(recipe_contents.replace(http_url, https_url))
                print(recipe_path)
                print("  Replaced: %s" % (http_url))
                print("      With: %s" % (https_url))
            else:
                print(recipe_path)
                print("   Replace: %s" % (http_url))
                print("      With: %s" % (https_url))
            return True
    except Exception as err:
        print("[ERROR] %s (%s)" % (recipe_path, err))
        raise err


def main():
    """Scans all recipes in the specified folder for HTTP URLs."""

    # Parse command line arguments.
    argparser = build_argument_parser()
    args = argparser.parse_args()

    if args.repo_path:
        path = args.repo_path
    else:
        path = "~/Library/AutoPkg/Recipes"

    # Process all recipes looking for HTTP URLs.
    suggestion_count = 0
    for dirpath, dirnames, filenames in os.walk(os.path.expanduser(path)):
        for dirname in dirnames:
            dirnames = [x for x in dirnames if not x.startswith(".")]
        filenames = [x for x in filenames if x.endswith(".recipe")]
        for filename in filenames:
            if process_recipe(os.path.join(dirpath, filename), args):
                suggestion_count += 1

    if not args.auto and suggestion_count > 0:
        changes = "change" if suggestion_count == 1 else "changes"
        print(
            "\n%d suggested %s. To apply, run again with --auto."
            % (suggestion_count, changes)
        )


if __name__ == "__main__":
    main()