rickheil/munkilinter

## munkilinter
#!/usr/bin/python
# encoding: utf8
#
#  Portions of code re-used from "makecatalogs" by Greg Neagle
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# Yo may obtain a copy of hte License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
munkilint

Created by Rick Heil

Recursively scans a directory, looking for installer item info files.
Runs a series of sanity checks on the found files, and reports any
errors. Almost all of this is to ensure that committed pkginfo files
conform to the standards of my org - your standards will almost certainly
require different checks!

Assumes that a pkgs, pkgsinfo, and icon directory are available in the current
working directory.

"""

import sys
import os
import plistlib
import hashlib

EXIT_CODE = 0

def print_utf8(text):
    '''Print Unicode text as UTF-8'''
    print text.encode('UTF-8')


def print_err_utf8(text):
    '''Print Unicode text to stderr as UTF-8'''
    print >> sys.stderr, text.encode('UTF-8')


def pkglint(pkgspath, pkgsinfopath, iconspath):
    '''Runs lint checks to ensure that pkginfo files confirm to our standards and
    are relatively error free.'''

    # clean slate
    errors = []
    global EXIT_CODE

    # walk through pkgsinfo files
    for dirpath, dirnames, filenames in os.walk(pkgsinfopath):
        for dirname in dirnames:
            # skip hidden directories
            if dirname.startswith('.'):
                dirnames.remove(dirname)

        for filename in filenames:
            # skip hidden files
            if filename.startswith('.'):
                continue

            filepath = os.path.join(dirpath, filename)

            print "Checking %s..." % filename

            # check pkginfo syntax by loading with plistlib - obvious syntax problems will
            # cause an error to be thrown.
            try:
                pkginfo = plistlib.readPlist(filepath)
            except IOError, inst:
                errors.append("IO error for %s: %s" % (pkgsinfopath, inst))
                EXIT_CODE = -1
                continue
            except TypeError, inst:
                errors.append("Unexpected error for %s: %s" % (pkgsinfopath, inst))
                EXIT_CODE = -1
                continue

            print "    Linting %s..." % filename

            # check there is a package name
            if not pkginfo.get('name'):
                errors.append("WARNING: file %s is missing name"
                              % filepath[len(pkgsinfopath)+1:])
                EXIT_CODE = -1
                continue

            # check there is a display name
            if not pkginfo.get('display_name'):
                errors.append("WARNING: file %s is missing name"
                              % filepath[len(pkgsinfopath)+1:])
                EXIT_CODE = -1
                continue

            # check admin notes are not blank
            if not pkginfo.get('notes'):
                errors.append("LINT ERROR: file %s is missing admin notes"
                              % filepath[len(pkgsinfopath)+1:])

            # check description is not blank
            if not pkginfo.get('description'):
                errors.append("LINT ERROR: file %s is missing description"
                              % filepath[len(pkgsinfopath)+1:])

            # check icon is present or available - either an icon should be specified,
            # or the name of the pkg should be a valid icon name too.
            if not pkginfo.get('icon_name'):
                defaulticonpath = os.path.join(iconspath, pkginfo.get('name') + '.png')
                if not os.path.isfile(defaulticonpath):
                    errors.append("LINT ERROR: file %s is missing an icon"
                                  % filepath[len(pkgsinfopath)+1:])
            else:
                # check icon exists and path does not have special characters in it.
                pkgiconpath = os.path.join(iconspath, pkginfo.get('icon_name'))
                if not os.path.isfile(pkgiconpath):
                    errors.append("LINT ERROR: file %s has invalid icon path specified"
                                  % filepath[len(pkgsinfopath)+1:])
                if set('~!@#$%^&*()+{}":;\']+$').intersection(pkgiconpath):
                    errors.append("LINT ERROR: file %s has special characters in icon file path"
                                  % filepath[len(pkgsinfopath)+1:])

            # check category is present
            if not pkginfo.get('category'):
                errors.append("LINT ERROR: file %s is missing a category"
                              % filepath[len(pkgsinfopath)+1:])
                # TODO - make this check on a whitelist of categories.

            # check if a developer is specified
            if not pkginfo.get('developer'):
                errors.append("LINT ERROR: file %s is missing a developer"
                              % filepath[len(pkgsinfopath)+1:])

            # check at least one catalog is specified
            if not pkginfo.get('catalogs'):
                errors.append("LINT ERROR: file %s is not listed in any catalogs"
                              % filepath[len(pkgsinfopath)+1:])

            # check installer item is present
            installer_type = pkginfo.get('installer_type')
            do_installer_check = True
            if installer_type in ['nopkg', 'apple_update_metadata']:
                # don't run for nopkg or metadata types
                do_installer_check = False
                print "    Skipping installer checks for %s" % filename

            if do_installer_check:
                print "    Running installer checks for %s" % filename
                installeritempath = os.path.join(
                    pkgspath, pkginfo['installer_item_location'])

                # check if item exists at location specified in pkginfo file
                if not os.path.isfile(installeritempath):
                    errors.append("ERROR: file %s has invalid path to installer item"
                                  % filepath[len(pkgsinfopath)+1:])
                    EXIT_CODE = -1

                # check if any special chars are in the path or name
                if set('~!@#$%^&*()+{}":;\']+$').intersection(installeritempath):
                    errors.append("LINT ERROR: file %s has special characters in installer "
                                  "item path" % filepath[len(pkgsinfopath)+1:])

                # check if hash of installer item matches the file in CI
                print "    Comparing file hashes for %s..." % os.path.basename(installeritempath)

                blocksize = 65536
                hasher = hashlib.sha256()
                try:
                    with open(installeritempath, 'rb') as installeritemfile:
                        readbuffer = installeritemfile.read(blocksize)
                        while len(readbuffer) > 0:
                            hasher.update(readbuffer)
                            readbuffer = installeritemfile.read(blocksize)

                    installeritemhash = hasher.hexdigest()

                    if installeritemhash != pkginfo.get('installer_item_hash'):
                        errors.append("ERROR: file %s has invalid installer item hash"
                                      % filepath[len(pkgsinfopath)+1:])
                        EXIT_CODE = -1

                except IOError, inst:
                    errors.append("ERROR: IOError reading %s" % filepath[len(pkgsinfopath)+1:])
                    EXIT_CODE = -1

            # check pkginfo file path does not have special characters
            if set('~!@#$%^&*()+{}":;\']+$').intersection(filepath):
                errors.append("LINT ERROR: special characters in pkginfo file path for %s:\n"
                              "Pkginfo file hash:          %s\n"
                              "Installer item actual hash: %s"
                              % (filepath[len(pkgsinfopath)+1:]),
                              pkginfo.get('installer_item_hash'),
                              installeritemhash)


            # TODO: add check that pkgsinfopath and item installer path match standards.

    if errors:
        # group all errors together at the end for better visibility
        print
        for error in errors:
            print_err_utf8(error)
    else:
        # print a nice thank you message
        print "\nNo lint or pkginfo syntax errors found."

    return


def manifestlint(manifestspath):
    '''Does basic lint checks on manifest files'''

    errors = []
    global EXIT_CODE

    print "Checking manifests files..."
    # walk through manifest files
    for dirpath, dirnames, filenames in os.walk(manifestspath):
        for dirname in dirnames:
            # skip hidden folders
            if dirname.startswith('.'):
                dirnames.remove(dirname)

        for filename in filenames:
            # skip hidden files
            if filename.startswith('.'):
                continue

            filepath = os.path.join(dirpath, filename)

            # simple check for correct manifest syntax by
            # looking if plist syntax is right.

            try:
                manifest = plistlib.readPlist(filepath)
            except IOError, inst:
                errors.append("IO error for %s: %s" % (filepath, inst))
                EXIT_CODE = -1
                continue
            except TypeError, inst:
                errors.append("Unexpected error for %s: %s" % (filepath, inst))
                EXIT_CODE = -1
                continue
            except:
                errors.append("Unknown error in %s" % filepath)
                EXIT_CODE = 01
                continue

            # Now we run some of the more cosmetic lint checks.
            # check that there is a display name. Use len because get just checks
            # if the tag is there!
            if not manifest.get('display_name'):
                # don't throw display name errors on templates
                if "template" not in filepath:
                    errors.append("LINT ERROR: manifest %s is missing display name"
                                  % filepath[len(manifestspath)+1:])

            # check there is one catalog set if the manifest is top level
            if not manifest.get('catalogs'):
                if "groups" not in filepath and "template" not in manifestspath:
                    # only throw catalog errors on top level manifests, and skip
                    # the template manifests.
                    errors.append("LINT ERROR: manifest %s has no catalogs set"
                                  % filepath[len(manifestspath)+1:])

    if errors:
        # group all errors together at the end for better visibility
        for error in errors:
            print_err_utf8(error)
    else:
        # print a nice thank you message
        print "\nNo lint or manifest syntax errors found."

    return


def main():
    '''Main'''

    global EXIT_CODE

    # make sure folders we need exist.
    pkgspath = os.path.join(os.getcwd(), 'pkgs')
    if not os.path.exists(pkgspath):
        print_err_utf8("Error: pkgs directory does not exist!")
        exit(-1)

    pkgsinfopath = os.path.join(os.getcwd(), 'pkgsinfo')
    if not os.path.exists(pkgsinfopath):
        print_err_utf8("Error: pkgsinfo directory does not exist!")
        exit(-1)

    iconspath = os.path.join(os.getcwd(), 'icons')
    if not os.path.exists(iconspath):
        print_err_utf8("Error: icons directory does not exist!")
        exit(-1)

    manifestspath = os.path.join(os.getcwd(), 'manifests')
    if not os.path.exists(manifestspath):
        print_err_utf8("Error: manifests directory does not exist!")
        exit(-1)

    # run lint for items
    pkglint(pkgspath, pkgsinfopath, iconspath)

    # run lint for manifests
    manifestlint(manifestspath)

    if EXIT_CODE != 0:
        exit(-1)
    else:
        exit(0)


if __name__ == '__main__':
    main()
	#!/usr/bin/python
	# encoding: utf8
	#
	# Portions of code re-used from "makecatalogs" by Greg Neagle
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# Yo may obtain a copy of hte License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	"""
	munkilint

	Created by Rick Heil

	Recursively scans a directory, looking for installer item info files.
	Runs a series of sanity checks on the found files, and reports any
	errors. Almost all of this is to ensure that committed pkginfo files
	conform to the standards of my org - your standards will almost certainly
	require different checks!

	Assumes that a pkgs, pkgsinfo, and icon directory are available in the current
	working directory.

	"""

	import sys
	import os
	import plistlib
	import hashlib

	EXIT_CODE = 0

	def print_utf8(text):
	'''Print Unicode text as UTF-8'''
	print text.encode('UTF-8')


	def print_err_utf8(text):
	'''Print Unicode text to stderr as UTF-8'''
	print >> sys.stderr, text.encode('UTF-8')


	def pkglint(pkgspath, pkgsinfopath, iconspath):
	'''Runs lint checks to ensure that pkginfo files confirm to our standards and
	are relatively error free.'''

	# clean slate
	errors = []
	global EXIT_CODE

	# walk through pkgsinfo files
	for dirpath, dirnames, filenames in os.walk(pkgsinfopath):
	for dirname in dirnames:
	# skip hidden directories
	if dirname.startswith('.'):
	dirnames.remove(dirname)

	for filename in filenames:
	# skip hidden files
	if filename.startswith('.'):
	continue

	filepath = os.path.join(dirpath, filename)

	print "Checking %s..." % filename

	# check pkginfo syntax by loading with plistlib - obvious syntax problems will
	# cause an error to be thrown.
	try:
	pkginfo = plistlib.readPlist(filepath)
	except IOError, inst:
	errors.append("IO error for %s: %s" % (pkgsinfopath, inst))
	EXIT_CODE = -1
	continue
	except TypeError, inst:
	errors.append("Unexpected error for %s: %s" % (pkgsinfopath, inst))
	EXIT_CODE = -1
	continue

	print " Linting %s..." % filename

	# check there is a package name
	if not pkginfo.get('name'):
	errors.append("WARNING: file %s is missing name"
	% filepath[len(pkgsinfopath)+1:])
	EXIT_CODE = -1
	continue

	# check there is a display name
	if not pkginfo.get('display_name'):
	errors.append("WARNING: file %s is missing name"
	% filepath[len(pkgsinfopath)+1:])
	EXIT_CODE = -1
	continue

	# check admin notes are not blank
	if not pkginfo.get('notes'):
	errors.append("LINT ERROR: file %s is missing admin notes"
	% filepath[len(pkgsinfopath)+1:])

	# check description is not blank
	if not pkginfo.get('description'):
	errors.append("LINT ERROR: file %s is missing description"
	% filepath[len(pkgsinfopath)+1:])

	# check icon is present or available - either an icon should be specified,
	# or the name of the pkg should be a valid icon name too.
	if not pkginfo.get('icon_name'):
	defaulticonpath = os.path.join(iconspath, pkginfo.get('name') + '.png')
	if not os.path.isfile(defaulticonpath):
	errors.append("LINT ERROR: file %s is missing an icon"
	% filepath[len(pkgsinfopath)+1:])
	else:
	# check icon exists and path does not have special characters in it.
	pkgiconpath = os.path.join(iconspath, pkginfo.get('icon_name'))
	if not os.path.isfile(pkgiconpath):
	errors.append("LINT ERROR: file %s has invalid icon path specified"
	% filepath[len(pkgsinfopath)+1:])
	if set('~!@#$%^&*()+{}":;\']+$').intersection(pkgiconpath):
	errors.append("LINT ERROR: file %s has special characters in icon file path"
	% filepath[len(pkgsinfopath)+1:])

	# check category is present
	if not pkginfo.get('category'):
	errors.append("LINT ERROR: file %s is missing a category"
	% filepath[len(pkgsinfopath)+1:])
	# TODO - make this check on a whitelist of categories.

	# check if a developer is specified
	if not pkginfo.get('developer'):
	errors.append("LINT ERROR: file %s is missing a developer"
	% filepath[len(pkgsinfopath)+1:])

	# check at least one catalog is specified
	if not pkginfo.get('catalogs'):
	errors.append("LINT ERROR: file %s is not listed in any catalogs"
	% filepath[len(pkgsinfopath)+1:])

	# check installer item is present
	installer_type = pkginfo.get('installer_type')
	do_installer_check = True
	if installer_type in ['nopkg', 'apple_update_metadata']:
	# don't run for nopkg or metadata types
	do_installer_check = False
	print " Skipping installer checks for %s" % filename

	if do_installer_check:
	print " Running installer checks for %s" % filename
	installeritempath = os.path.join(
	pkgspath, pkginfo['installer_item_location'])

	# check if item exists at location specified in pkginfo file
	if not os.path.isfile(installeritempath):
	errors.append("ERROR: file %s has invalid path to installer item"
	% filepath[len(pkgsinfopath)+1:])
	EXIT_CODE = -1

	# check if any special chars are in the path or name
	if set('~!@#$%^&*()+{}":;\']+$').intersection(installeritempath):
	errors.append("LINT ERROR: file %s has special characters in installer "
	"item path" % filepath[len(pkgsinfopath)+1:])

	# check if hash of installer item matches the file in CI
	print " Comparing file hashes for %s..." % os.path.basename(installeritempath)

	blocksize = 65536
	hasher = hashlib.sha256()
	try:
	with open(installeritempath, 'rb') as installeritemfile:
	readbuffer = installeritemfile.read(blocksize)
	while len(readbuffer) > 0:
	hasher.update(readbuffer)
	readbuffer = installeritemfile.read(blocksize)

	installeritemhash = hasher.hexdigest()

	if installeritemhash != pkginfo.get('installer_item_hash'):
	errors.append("ERROR: file %s has invalid installer item hash"
	% filepath[len(pkgsinfopath)+1:])
	EXIT_CODE = -1

	except IOError, inst:
	errors.append("ERROR: IOError reading %s" % filepath[len(pkgsinfopath)+1:])
	EXIT_CODE = -1

	# check pkginfo file path does not have special characters
	if set('~!@#$%^&*()+{}":;\']+$').intersection(filepath):
	errors.append("LINT ERROR: special characters in pkginfo file path for %s:\n"
	"Pkginfo file hash: %s\n"
	"Installer item actual hash: %s"
	% (filepath[len(pkgsinfopath)+1:]),
	pkginfo.get('installer_item_hash'),
	installeritemhash)


	# TODO: add check that pkgsinfopath and item installer path match standards.

	if errors:
	# group all errors together at the end for better visibility
	print
	for error in errors:
	print_err_utf8(error)
	else:
	# print a nice thank you message
	print "\nNo lint or pkginfo syntax errors found."

	return


	def manifestlint(manifestspath):
	'''Does basic lint checks on manifest files'''

	errors = []
	global EXIT_CODE

	print "Checking manifests files..."
	# walk through manifest files
	for dirpath, dirnames, filenames in os.walk(manifestspath):
	for dirname in dirnames:
	# skip hidden folders
	if dirname.startswith('.'):
	dirnames.remove(dirname)

	for filename in filenames:
	# skip hidden files
	if filename.startswith('.'):
	continue

	filepath = os.path.join(dirpath, filename)

	# simple check for correct manifest syntax by
	# looking if plist syntax is right.

	try:
	manifest = plistlib.readPlist(filepath)
	except IOError, inst:
	errors.append("IO error for %s: %s" % (filepath, inst))
	EXIT_CODE = -1
	continue
	except TypeError, inst:
	errors.append("Unexpected error for %s: %s" % (filepath, inst))
	EXIT_CODE = -1
	continue
	except:
	errors.append("Unknown error in %s" % filepath)
	EXIT_CODE = 01
	continue

	# Now we run some of the more cosmetic lint checks.
	# check that there is a display name. Use len because get just checks
	# if the tag is there!
	if not manifest.get('display_name'):
	# don't throw display name errors on templates
	if "template" not in filepath:
	errors.append("LINT ERROR: manifest %s is missing display name"
	% filepath[len(manifestspath)+1:])

	# check there is one catalog set if the manifest is top level
	if not manifest.get('catalogs'):
	if "groups" not in filepath and "template" not in manifestspath:
	# only throw catalog errors on top level manifests, and skip
	# the template manifests.
	errors.append("LINT ERROR: manifest %s has no catalogs set"
	% filepath[len(manifestspath)+1:])

	if errors:
	# group all errors together at the end for better visibility
	for error in errors:
	print_err_utf8(error)
	else:
	# print a nice thank you message
	print "\nNo lint or manifest syntax errors found."

	return


	def main():
	'''Main'''

	global EXIT_CODE

	# make sure folders we need exist.
	pkgspath = os.path.join(os.getcwd(), 'pkgs')
	if not os.path.exists(pkgspath):
	print_err_utf8("Error: pkgs directory does not exist!")
	exit(-1)

	pkgsinfopath = os.path.join(os.getcwd(), 'pkgsinfo')
	if not os.path.exists(pkgsinfopath):
	print_err_utf8("Error: pkgsinfo directory does not exist!")
	exit(-1)

	iconspath = os.path.join(os.getcwd(), 'icons')
	if not os.path.exists(iconspath):
	print_err_utf8("Error: icons directory does not exist!")
	exit(-1)

	manifestspath = os.path.join(os.getcwd(), 'manifests')
	if not os.path.exists(manifestspath):
	print_err_utf8("Error: manifests directory does not exist!")
	exit(-1)

	# run lint for items
	pkglint(pkgspath, pkgsinfopath, iconspath)

	# run lint for manifests
	manifestlint(manifestspath)

	if EXIT_CODE != 0:
	exit(-1)
	else:
	exit(0)


	if __name__ == '__main__':
	main()