munkilinter

#!/usr/bin/python
# encoding: utf8
#
#  Portions of code re-used from "makecatalogs" by Greg Neagle
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# Yo may obtain a copy of hte License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
munkilint

Created by Rick Heil

Recursively scans a directory, looking for installer item info files.
Runs a series of sanity checks on the found files, and reports any
errors. Almost all of this is to ensure that committed pkginfo files
conform to the standards of my org - your standards will almost certainly
require different checks!

Assumes that a pkgs, pkgsinfo, and icon directory are available in the current
working directory.

"""

import sys
import os
import plistlib
import hashlib

EXIT_CODE = 0

def print_utf8(text):
    '''Print Unicode text as UTF-8'''
    print text.encode('UTF-8')


def print_err_utf8(text):
    '''Print Unicode text to stderr as UTF-8'''
    print >> sys.stderr, text.encode('UTF-8')


def pkglint(pkgspath, pkgsinfopath, iconspath):
    '''Runs lint checks to ensure that pkginfo files confirm to our standards and
    are relatively error free.'''

    # clean slate
    errors = []
    global EXIT_CODE

    # walk through pkgsinfo files
    for dirpath, dirnames, filenames in os.walk(pkgsinfopath):
        for dirname in dirnames:
            # skip hidden directories
            if dirname.startswith('.'):
                dirnames.remove(dirname)

        for filename in filenames:
            # skip hidden files
            if filename.startswith('.'):
                continue

            filepath = os.path.join(dirpath, filename)

            print "Checking %s..." % filename

            # check pkginfo syntax by loading with plistlib - obvious syntax problems will
            # cause an error to be thrown.
            try:
                pkginfo = plistlib.readPlist(filepath)
            except IOError, inst:
                errors.append("IO error for %s: %s" % (pkgsinfopath, inst))
                EXIT_CODE = -1
                continue
            except TypeError, inst:
                errors.append("Unexpected error for %s: %s" % (pkgsinfopath, inst))
                EXIT_CODE = -1
                continue

            print "    Linting %s..." % filename

            # check there is a package name
            if not pkginfo.get('name'):
                errors.append("WARNING: file %s is missing name"
                              % filepath[len(pkgsinfopath)+1:])
                EXIT_CODE = -1
                continue

            # check there is a display name
            if not pkginfo.get('display_name'):
                errors.append("WARNING: file %s is missing name"
                              % filepath[len(pkgsinfopath)+1:])
                EXIT_CODE = -1
                continue

            # check admin notes are not blank
            if not pkginfo.get('notes'):
                errors.append("LINT ERROR: file %s is missing admin notes"
                              % filepath[len(pkgsinfopath)+1:])

            # check description is not blank
            if not pkginfo.get('description'):
                errors.append("LINT ERROR: file %s is missing description"
                              % filepath[len(pkgsinfopath)+1:])

            # check icon is present or available - either an icon should be specified,
            # or the name of the pkg should be a valid icon name too.
            if not pkginfo.get('icon_name'):
                defaulticonpath = os.path.join(iconspath, pkginfo.get('name') + '.png')
                if not os.path.isfile(defaulticonpath):
                    errors.append("LINT ERROR: file %s is missing an icon"
                                  % filepath[len(pkgsinfopath)+1:])
            else:
                # check icon exists and path does not have special characters in it.
                pkgiconpath = os.path.join(iconspath, pkginfo.get('icon_name'))
                if not os.path.isfile(pkgiconpath):
                    errors.append("LINT ERROR: file %s has invalid icon path specified"
                                  % filepath[len(pkgsinfopath)+1:])
                if set('~!@#$%^&*()+{}":;\']+$').intersection(pkgiconpath):
                    errors.append("LINT ERROR: file %s has special characters in icon file path"
                                  % filepath[len(pkgsinfopath)+1:])

            # check category is present
            if not pkginfo.get('category'):
                errors.append("LINT ERROR: file %s is missing a category"
                              % filepath[len(pkgsinfopath)+1:])
                # TODO - make this check on a whitelist of categories.

            # check if a developer is specified
            if not pkginfo.get('developer'):
                errors.append("LINT ERROR: file %s is missing a developer"
                              % filepath[len(pkgsinfopath)+1:])

            # check at least one catalog is specified
            if not pkginfo.get('catalogs'):
                errors.append("LINT ERROR: file %s is not listed in any catalogs"
                              % filepath[len(pkgsinfopath)+1:])

            # check installer item is present
            installer_type = pkginfo.get('installer_type')
            do_installer_check = True
            if installer_type in ['nopkg', 'apple_update_metadata']:
                # don't run for nopkg or metadata types
                do_installer_check = False
                print "    Skipping installer checks for %s" % filename

            if do_installer_check:
                print "    Running installer checks for %s" % filename
                installeritempath = os.path.join(
                    pkgspath, pkginfo['installer_item_location'])

                # check if item exists at location specified in pkginfo file
                if not os.path.isfile(installeritempath):
                    errors.append("ERROR: file %s has invalid path to installer item"
                                  % filepath[len(pkgsinfopath)+1:])
                    EXIT_CODE = -1

                # check if any special chars are in the path or name
                if set('~!@#$%^&*()+{}":;\']+$').intersection(installeritempath):
                    errors.append("LINT ERROR: file %s has special characters in installer "
                                  "item path" % filepath[len(pkgsinfopath)+1:])

                # check if hash of installer item matches the file in CI
                print "    Comparing file hashes for %s..." % os.path.basename(installeritempath)

                blocksize = 65536
                hasher = hashlib.sha256()
                try:
                    with open(installeritempath, 'rb') as installeritemfile:
                        readbuffer = installeritemfile.read(blocksize)
                        while len(readbuffer) > 0:
                            hasher.update(readbuffer)
                            readbuffer = installeritemfile.read(blocksize)
                        
                    installeritemhash = hasher.hexdigest()

                    if installeritemhash != pkginfo.get('installer_item_hash'):
                        errors.append("ERROR: file %s has invalid installer item hash"
                                      % filepath[len(pkgsinfopath)+1:])
                        EXIT_CODE = -1

                except IOError, inst:
                    errors.append("ERROR: IOError reading %s" % filepath[len(pkgsinfopath)+1:])
                    EXIT_CODE = -1

            # check pkginfo file path does not have special characters
            if set('~!@#$%^&*()+{}":;\']+$').intersection(filepath):
                errors.append("LINT ERROR: special characters in pkginfo file path for %s:\n"
                              "Pkginfo file hash:          %s\n"
                              "Installer item actual hash: %s"
                              % (filepath[len(pkgsinfopath)+1:]),
                              pkginfo.get('installer_item_hash'),
                              installeritemhash)


            # TODO: add check that pkgsinfopath and item installer path match standards.

    if errors:
        # group all errors together at the end for better visibility
        print
        for error in errors:
            print_err_utf8(error)
    else:
        # print a nice thank you message
        print "\nNo lint or pkginfo syntax errors found."

    return


def manifestlint(manifestspath):
    '''Does basic lint checks on manifest files'''

    errors = []
    global EXIT_CODE

    print "Checking manifests files..."
    # walk through manifest files
    for dirpath, dirnames, filenames in os.walk(manifestspath):
        for dirname in dirnames:
            # skip hidden folders
            if dirname.startswith('.'):
                dirnames.remove(dirname)

        for filename in filenames:
            # skip hidden files
            if filename.startswith('.'):
                continue

            filepath = os.path.join(dirpath, filename)

            # simple check for correct manifest syntax by
            # looking if plist syntax is right.

            try:
                manifest = plistlib.readPlist(filepath)
            except IOError, inst:
                errors.append("IO error for %s: %s" % (filepath, inst))
                EXIT_CODE = -1
                continue
            except TypeError, inst:
                errors.append("Unexpected error for %s: %s" % (filepath, inst))
                EXIT_CODE = -1
                continue
            except:
                errors.append("Unknown error in %s" % filepath)
                EXIT_CODE = 01
                continue

            # Now we run some of the more cosmetic lint checks.
            # check that there is a display name. Use len because get just checks
            # if the tag is there!
            if not manifest.get('display_name'):
                # don't throw display name errors on templates
                if "template" not in filepath:
                    errors.append("LINT ERROR: manifest %s is missing display name"
                                  % filepath[len(manifestspath)+1:])

            # check there is one catalog set if the manifest is top level
            if not manifest.get('catalogs'):
                if "groups" not in filepath and "template" not in manifestspath:
                    # only throw catalog errors on top level manifests, and skip
                    # the template manifests.
                    errors.append("LINT ERROR: manifest %s has no catalogs set"
                                  % filepath[len(manifestspath)+1:])

    if errors:
        # group all errors together at the end for better visibility
        for error in errors:
            print_err_utf8(error)
    else:
        # print a nice thank you message
        print "\nNo lint or manifest syntax errors found."

    return


def main():
    '''Main'''

    global EXIT_CODE

    # make sure folders we need exist.
    pkgspath = os.path.join(os.getcwd(), 'pkgs')
    if not os.path.exists(pkgspath):
        print_err_utf8("Error: pkgs directory does not exist!")
        exit(-1)

    pkgsinfopath = os.path.join(os.getcwd(), 'pkgsinfo')
    if not os.path.exists(pkgsinfopath):
        print_err_utf8("Error: pkgsinfo directory does not exist!")
        exit(-1)

    iconspath = os.path.join(os.getcwd(), 'icons')
    if not os.path.exists(iconspath):
        print_err_utf8("Error: icons directory does not exist!")
        exit(-1)

    manifestspath = os.path.join(os.getcwd(), 'manifests')
    if not os.path.exists(manifestspath):
        print_err_utf8("Error: manifests directory does not exist!")
        exit(-1)

    # run lint for items
    pkglint(pkgspath, pkgsinfopath, iconspath)

    # run lint for manifests
    manifestlint(manifestspath)

    if EXIT_CODE != 0:
        exit(-1)
    else:
        exit(0)


if __name__ == '__main__':
    main()