wm_enterprise_downloader_test.py

#!/usr/bin/python3
# encoding: utf-8
'''
download Wikimedia Enterprise HTML dumps for all or for specified wiki projects
'''
import os
import getopt
import logging
import sys
import time

from wm_enterprise_downloader import AuthManager
from wm_enterprise_downloader import get_settings
from wm_enterprise_downloader import setup_logging


LOG = logging.getLogger(__name__)

def usage(message=None):
    '''display usage info about this script'''
    if message is not None:
        print(message)
    usage_message = """Usage: wm_enterprise_downloader_test.py
         [--creds path-to-creds-file] [--settings path-to-settings-file]
         [--dryrun] [--verbose]| --help

Arguments:

  --creds     (-c):  path to plain text file with HTTP Basic Auth credentials
                     format: two lines, varname=value, with the varnames user and passwd
                             blank lines and those starting with '#' are skipped
                     default: .wm_enterprise_creds in current working directory
  --settings  (-s):  path to settings file
                     format: two lines, varname=value; see the sample file in this repo
                             for each setting and its default value
                             blank lines and those starting with '#' are skipped
                     default: wm_enterprise_downloader_settings in current working directory
  --dryrun    (-d):  don't download any wiki dumps but print the list of urls for retrieval
  --verbose   (-v):  show some progress messages while running
  --help      (-h):  display this usage message

Example usage:

   python ./wm_enterprise_downloader_test.py --verbose

"""
    print(usage_message)
    sys.exit(1)


def fillin_args(options, args):
    '''
    walk through all the provided options and stuff the appropriate values in the args
    '''
    for (opt, val) in options:
        if opt in ["-c", "--creds"]:
            args['creds'] = val
        elif opt in ["-s", "--settings"]:
            args['settings'] = val
        elif opt in ["-d", "--dryrun"]:
            args['dryrun'] = True
        elif opt in ["-v", "--verbose"]:
            args['verbose'] = True
        elif opt in ["-h", "--help"]:
            usage('Help for this script')
        else:
            usage("Unknown option specified: <%s>" % opt)


def get_args():
    '''
    get and validate command-line args and return them
    '''
    try:
        (options, remainder) = getopt.gnu_getopt(
            sys.argv[1:], "c:s:dvh", [
                "creds=", "settings=",
                "dryrun", "verbose", "help"])

    except getopt.GetoptError as err:
        usage("Unknown option specified: " + str(err))

    args = {'dryrun': False, 'verbose': False}
    args['settings'] = os.path.join(os.getcwd(), 'wm_enterprise_downloader_settings')
    args['creds'] = os.path.join(os.getcwd(), '.wm_enterprise_creds')

    fillin_args(options, args)

    if remainder:
        usage("Unknown option(s) specified: {opt}".format(opt=remainder[0]))

    return args


def do_main():
    '''
    entry point

    make sure we can get an auth token and refresh it after 24 hours when it expires.
    that is all.
    '''
    args = get_args()
    setup_logging(args)
    settings = get_settings(args)
    auth_mgr = AuthManager(args['creds'], settings)
    print(auth_mgr.auth_tokens['access'])
    # sleep a little more than 24 hours
    # time.sleep(86400 + 30)
    time.sleep(300)
    auth_mgr.refresh()
    print("after refresh:", auth_mgr.auth_tokens['access'])
    LOG.info("done!")


if __name__ == '__main__':
    do_main()