Webifier validation script

README.md

webifier

Script to test if sites are visible to users on the open internet.

If they are, the don't need to be "webified" to be sent through FirePass device.

Dependencies

The script depends on the requests library to perform HTTP tests.

pip install .

Usage

webifier --help

Input file

File should be tab-separated file with URL in the 3rd column.

Do not include headers in the file.

Sample entry

62166	ICT service desk	https://detwww.det.nsw.edu.au/it/ictservicedesk/index.htm	webifier

setup.py

from setuptools import setup, find_packages

setup(
    name="webifier",
    author='Gary Reynolds',
    author_email='gary.reynolds@touchtechnology.com.au',
    py_modules={'': 'webifier'},
    install_requires=[
        'click',
        'requests',
    ],
    entry_points={
        'console_scripts': [
            'webifier = webifier:webifier',
        ],
    },
    classifiers=[
        'License :: Other/Proprietary License',
        'Operating System :: POSIX :: Linux',
        'Intended Audience :: System Administrators',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Topic :: System',
        'Topic :: Utilities',
    ],
)

webifier.py

import csv
import logging
import sys

import click
import requests


def is_globally_visible(url, timeout):
    """
    Check if the URL is accessible on an external internet connection.

    :param url: the URL to check
    :param timeout: number of seconds before timing out
    :returns: boolean status
    """
    res = requests.get(url, timeout=timeout)
    res.raise_for_status()
    return res.status_code


@click.command()
@click.pass_context
@click.argument('infile', metavar='infile', type=click.File(mode='rb'))
@click.argument('outfile', metavar='outfile', type=click.File(mode='wb'))
@click.option('-t', '--timeout', metavar='seconds',
              help='Seconds to wait for HTTP response',
              type=int, default=15, show_default=True)
@click.option('-l', '--logfile', metavar='filename', type=click.File(mode='a'))
@click.option('--debug', is_flag=True, help='Enable debugging logs')
def webifier(ctx, infile, outfile, timeout, logfile, debug):
    """
    webifier

    Check if resources listed in the DEC Insight Portal with "webifier" code
    really require it. If the service is already globally accessible, this adds
    an attack vector where a malicious user could intercept the user session.
    """
    # Configure logging
    ctx.log = logging.getLogger(ctx.info_name)
    ctx.log.setLevel(logging.DEBUG if debug else logging.WARNING)

    handler = logging.StreamHandler(logfile)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    handler.setFormatter(formatter)
    ctx.log.addHandler(handler)

    # Log how the process was instantiated
    ctx.log.debug('infile=%r', infile)
    ctx.log.debug('outfile=%r', outfile)
    ctx.log.debug('timeout=%r', timeout)
    ctx.log.debug('logfile=%r', logfile)

    # instantiate a csv writer to echo the input line to if it is a match
    writer = csv.writer(outfile, dialect=csv.excel_tab)

    for row in csv.reader(infile, dialect=csv.excel_tab):
        # unpack the csv row, headings would be useful and then we could use a
        # DictReader instead.
        pk, label, url, tag = row
        try:
            visible = is_globally_visible(url, timeout)
        except Exception:
            # if something went wrong we'll assume the worst and leave it as
            # requiring webification; later run of this tool may pick it up
            # again.
            visible = False

            # log failure for manual verification - could be a dead link
            ctx.log.exception('Manual verification required: %r', url)

        if visible:
            writer.writerow(row)


if __name__ == '__main__':
    webifier()