CLI

main.py

# Truncated for Medium, see github.com/bitdotioinc/simple-pipeline 

if __name__ == '__main__':
    # Parse command line options and arguments
    opts = [opt[1:] for opt in sys.argv[1:] if opt.startswith("-")]
    local_source = 'local_source' in opts
    opts = [opt for opt in opts if opt != 'local_source']
    args = [arg for arg in sys.argv[1:] if not arg.startswith("-")]
    # Validation
    if len(args) != len(opts) + 2:
        raise ValueError("At least one argument is missing, check the README.")
    # Set up local variables
    source, destination = args[-2:]
    opt_args = args[:-2]
    option_args = dict(zip(opts, opt_args))

    # Execute ETL
    main(source, destination, local_source, option_args)