handle_input_output.py

#!/use/bin/env python3
#
# Generic script for handling input and output files in parallel
#
# Author: Werner Robitza
import argparse
import os
from tqdm import tqdm
from multiprocessing import Pool

__version__ = "0.1"


def create_parser():
    parser = argparse.ArgumentParser(
        prog="Program Name",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        epilog=""
    )
    parser.add_argument(
        'input',
        nargs='+',
        help='input files'
    )
    parser.add_argument(
        '-o', '--output',
        help='output directory',
        required=True
    )
    parser.add_argument(
        '-f', '--force',
        action='store_true',
        help="Force overwrite existing files"
    )
    parser.add_argument(
        '-d', '--debug',
        action='store_true',
        help="Print debugging output"
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help="Print verbose output"
    )
    parser.add_argument(
        '-n', '--dry-run',
        action='store_true',
        help="Do not run, only print what would be done"
    )
    parser.add_argument(
        '--version',
        action='version',
        version='%(prog)s v{}'.format(__version__),
        help="Print version and exit"
    )
    parser.add_argument(
        '-c', '--concurrency',
        type=int,
        default=4,
        help="Number of parallel processes"
    )
    return parser


def action(args):
    input_file, output_file, force, debug, verbose, dry_run = args
    pass


def main():
    parser = create_parser()
    cli_args = parser.parse_args()

    all_args = []

    for input_file in cli_args.input:
        output_file = os.path.join(
            cli_args.output,
            os.path.basename(input_file)
        )
        all_args.append((input_file, output_file, cli_args.force, cli_args.debug, cli_args.verbose, cli_args.dry_run))

    if cli_args.concurrency > 1:
        pool = Pool(processes=cli_args.concurrency)
        for _ in tqdm(pool.imap_unordered(action, all_args), total=len(all_args), disable=cli_args.debug):
            pass
    else:
        for args in tqdm(all_args):
            action(args)


if __name__ == '__main__':
    main()