HTTPS Spotter

https_spotter.py

#!/usr/local/autopkg/python

# HTTPS Spotter
# Copyright 2016-2025 Elliot Jordan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""https_spotter.py.

Scans for HTTP URLs in a specified repository of AutoPkg recipes and suggests
HTTPS alternatives wherever possible.
"""

import argparse
import os
import plistlib
import subprocess

import yaml

# Path to curl.
CURL_PATH = "/usr/bin/curl"


def build_argument_parser():
    """Build and return the argument parser."""

    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        "repo_path", nargs="?", help="Path to search for AutoPkg recipes."
    )
    parser.add_argument(
        "--auto",
        action="store_true",
        default=False,
        help="Automatically apply suggested changes to recipes. Only recommended "
        "for repos you manage or are submitting a pull request to. (Applying "
        "changes to repos added using `autopkg repo-add` may result in failure "
        "to update the repo in the future.)",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        help="Print additional output useful for "
        "troubleshooting. (Can be specified multiple times.)",
    )
    return parser


def check_url(url, verbose):
    """Checks a given HTTPS URL to see whether an HTTPS equivalent is
    available."""

    https_url = url.replace("http:", "https:")

    try:
        cmd = [
            CURL_PATH,
            "--head",
            "--silent",
            "--location",
            "--max-time",
            "10",
            "--max-redirs",
            "10",
            "--url",
            https_url,
        ]
        if verbose > 1:
            print(f"  Curl command: {' '.join(cmd)}")
        proc = subprocess.run(cmd, check=False, capture_output=True, timeout=15)
        http_response = [x for x in proc.stdout.splitlines() if x.startswith(b"HTTP/")]
        if not http_response:
            return False
        http_status = int(http_response[0].split()[1])
        if verbose > 2:
            print(f"  Exit code: {proc.returncode}")
            print(f"  HTTP status: {http_status}")
        if proc.returncode == 0:
            # Ignoring HTTP status because 404 with SSL is better than
            # 404 without.
            return https_url
    except (subprocess.TimeoutExpired, ValueError, IndexError) as err:
        if verbose > 3:
            print(f"  {err}")
    except Exception as err:
        if verbose > 3:
            print(f"  Unexpected error: {err}")

    return False


def process_recipe(recipe_path, args):
    """Processes one recipe."""

    try:
        # Read recipe.
        if not os.path.isfile(recipe_path):
            return False
        if recipe_path.endswith(".recipe.yaml"):
            with open(recipe_path, encoding="utf-8") as openfile:
                recipe = yaml.safe_load(openfile)
        else:
            with open(recipe_path, "rb") as openfile:
                recipe = plistlib.load(openfile)

        if not recipe:
            if args.verbose > 0:
                print(f"Skipping: {recipe_path} (empty or invalid recipe)")
            return False

        if "Process" in recipe:
            processors = [x.get("Processor") for x in recipe["Process"]]
            if "DeprecationWarning" in processors:
                if args.verbose > 0:
                    print(f"Skipping: {recipe_path} " f"(contains DeprecationWarning)")
                return False
            if args.verbose > 0:
                print(f"Processing: {recipe_path}")

        urls_to_check = set()

        # Gather HTTP URLs from Input.
        input_section = recipe.get("Input")
        if input_section:
            for value in input_section.values():
                if isinstance(value, str) and value.startswith("http:"):
                    urls_to_check.add(value)

        # Gather HTTP URLs from Processors.
        for proc in recipe.get("Process", []):
            if not proc.get("Arguments"):
                continue
            for _, value in proc.get("Arguments", {}).items():
                if not isinstance(value, str):
                    # Only looking at string arguments, not diving
                    # deeper into lists/dicts.
                    continue
                if value.startswith("http:"):
                    urls_to_check.add(value)

        # Check HTTP URLs for HTTPS equivalents.
        changes_made = False
        for http_url in urls_to_check:
            https_url = check_url(http_url, args.verbose)
            if not https_url:
                continue
            if args.auto:
                # Read/write as text instead of using plistlib in order
                # to prevent unrelated changes (e.g. whitespace) in diff.
                with open(recipe_path, encoding="utf-8") as openfile:
                    recipe_contents = openfile.read()
                with open(recipe_path, "w", encoding="utf-8") as openfile:
                    openfile.write(recipe_contents.replace(http_url, https_url))
                print(recipe_path)
                print(f"  Replaced: {http_url}")
                print(f"      With: {https_url}")
            else:
                print(recipe_path)
                print(f"   Replace: {http_url}")
                print(f"      With: {https_url}")
            changes_made = True
        return changes_made
    except Exception as err:
        print(f"[ERROR] {recipe_path} ({err})")
        if args.verbose > 3:
            import traceback

            traceback.print_exc()
        return False


def main():
    """Scans all recipes in the specified folder for HTTP URLs."""

    # Parse command line arguments.
    argparser = build_argument_parser()
    args = argparser.parse_args()

    if args.repo_path:
        path = args.repo_path
    else:
        path = "~/Library/AutoPkg/Recipes"

    # Process all recipes looking for HTTP URLs.
    suggestion_count = 0
    for dirpath, dirnames, filenames in os.walk(os.path.expanduser(path)):
        # Remove hidden directories
        dirnames[:] = [x for x in dirnames if not x.startswith(".")]
        filenames = [
            x for x in filenames if x.endswith(".recipe") or x.endswith(".recipe.yaml")
        ]
        for filename in filenames:
            if process_recipe(os.path.join(dirpath, filename), args):
                suggestion_count += 1

    if not args.auto and suggestion_count > 0:
        changes = "change" if suggestion_count == 1 else "changes"
        print(
            f"\n{suggestion_count} suggested {changes}. "
            f"To apply, run again with --auto."
        )


if __name__ == "__main__":
    main()

requirements.txt

PyYAML