HTTPS Spotter
#!/usr/local/autopkg/python | |
# encoding: utf-8 | |
# HTTPS Spotter | |
# Copyright 2016-2020 Elliot Jordan | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
"""https_spotter.py. | |
Scans for HTTP URLs in a specified repository of AutoPkg recipes and suggests | |
HTTPS alternatives wherever possible. | |
""" | |
import argparse | |
import os | |
import plistlib | |
import subprocess | |
# Path to curl. | |
CURL_PATH = "/usr/bin/curl" | |
def build_argument_parser(): | |
"""Build and return the argument parser.""" | |
parser = argparse.ArgumentParser( | |
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter | |
) | |
parser.add_argument( | |
"repo_path", help="Path to search for AutoPkg recipes." | |
) | |
parser.add_argument( | |
"--auto", | |
action="store_true", | |
default=False, | |
help="Automatically apply suggested changes to recipes. Only recommended " | |
"for repos you manage or are submitting a pull request to. (Applying " | |
"changes to repos added using `autopkg repo-add` may result in failure " | |
"to update the repo in the future.)", | |
) | |
parser.add_argument( | |
"-v", | |
"--verbose", | |
action="count", | |
default=0, | |
help="Print additional output useful for " | |
"troubleshooting. (Can be specified multiple times.)", | |
) | |
return parser | |
def check_url(url, verbose): | |
"""Checks a given HTTPS URL to see whether an HTTPS equivalent is | |
available.""" | |
https_url = url.replace("http:", "https:") | |
try: | |
cmd = [ | |
CURL_PATH, | |
"--head", | |
"--silent", | |
"--location", | |
"--max-time", | |
"5", | |
"--max-redirs", | |
"10", | |
"--url", | |
https_url, | |
] | |
if verbose > 1: | |
print(" Curl command: %s" % " ".join(cmd)) | |
proc = subprocess.run(cmd, check=False, capture_output=True) | |
http_response = [x for x in proc.stdout.splitlines() if x.startswith(b"HTTP/")] | |
http_status = int(http_response[0].split()[1]) | |
if verbose > 2: | |
print(" Exit code: %s" % proc.returncode) | |
print(" HTTP status: %s" % http_status) | |
if proc.returncode == 0: | |
# Ignoring HTTP status because 404 with SSL is better than 404 without. | |
return https_url | |
except Exception as err: | |
if verbose > 3: | |
print(" %s" % err) | |
return False | |
def process_recipe(recipe_path, args): | |
"""Processes one recipe.""" | |
try: | |
# Read recipe. | |
with open(recipe_path, "rb") as openfile: | |
recipe = plistlib.load(openfile) | |
if "Process" in recipe: | |
processors = [x.get("Processor") for x in recipe["Process"]] | |
if "DeprecationWarning" in processors: | |
if args.verbose > 0: | |
print("Skipping: %s (contains DeprecationWarning)" % (recipe_path)) | |
return | |
if args.verbose > 0: | |
print("Processing: %s" % (recipe_path)) | |
urls_to_check = [] | |
# Gather HTTP URLs from Input. | |
for key in recipe.get("Input", {}): | |
if isinstance(recipe["Input"][key], str) and recipe["Input"][ | |
key | |
].startswith("http:"): | |
urls_to_check.append(recipe["Input"][key]) | |
# Gather HTTP URLs from Processors. | |
for proc in recipe.get("Process", []): | |
for _, value in proc.get("Arguments", {}).items(): | |
if not isinstance(value, str): | |
# Only looking at string arguments, not diving | |
# deeper into lists/dicts. | |
continue | |
if value.startswith("http:"): | |
urls_to_check.append(value) | |
# Check HTTP URLs for HTTPS equivalents. | |
for http_url in urls_to_check: | |
https_url = check_url(http_url, args.verbose) | |
if not https_url: | |
continue | |
if args.auto: | |
# Read/write as text instead of using plistlib in order | |
# to prevent unrelated changes (e.g. whitespace) in diff. | |
with open(recipe_path, "r") as openfile: | |
recipe_contents = openfile.read() | |
with open(recipe_path, "w") as openfile: | |
openfile.write(recipe_contents.replace(http_url, https_url)) | |
print(recipe_path) | |
print(" Replaced: %s" % (http_url)) | |
print(" With: %s" % (https_url)) | |
else: | |
print(recipe_path) | |
print(" Replace: %s" % (http_url)) | |
print(" With: %s" % (https_url)) | |
return True | |
except Exception as err: | |
print("[ERROR] %s (%s)" % (recipe_path, err)) | |
raise err | |
def main(): | |
"""Scans all recipes in the specified folder for HTTP URLs.""" | |
# Parse command line arguments. | |
argparser = build_argument_parser() | |
args = argparser.parse_args() | |
if args.repo_path: | |
path = args.repo_path | |
else: | |
path = "~/Library/AutoPkg/Recipes" | |
# Process all recipes looking for HTTP URLs. | |
suggestion_count = 0 | |
for dirpath, dirnames, filenames in os.walk(os.path.expanduser(path)): | |
for dirname in dirnames: | |
dirnames = [x for x in dirnames if not x.startswith(".")] | |
filenames = [x for x in filenames if x.endswith(".recipe")] | |
for filename in filenames: | |
if process_recipe(os.path.join(dirpath, filename), args): | |
suggestion_count += 1 | |
if not args.auto and suggestion_count > 0: | |
changes = "change" if suggestion_count == 1 else "changes" | |
print( | |
"\n%d suggested %s. To apply, run again with --auto." | |
% (suggestion_count, changes) | |
) | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment