Benchling decaffeinate wrapper script

decaffeinate.py

#!/usr/bin/env python
import argparse
from collections import Counter, namedtuple
import json
import os
import subprocess
import sys
import textwrap
import urllib
import webbrowser


def main():
    if not os.path.exists('./scripts/dev/decaffeinate.py'):
        raise CLIException('This script must be run from the aurelia root.')
    if not os.path.exists('./node_modules/.bin/decaffeinate'):
        raise CLIException('decaffeinate not detected. You may need to run "npm install".')
    if not os.path.exists('./node_modules/.bin/jscodeshift'):
        raise CLIException('jscodeshift not detected. You may need to run "npm install".')

    parser = argparse.ArgumentParser(description=textwrap.dedent("""
        Benchling wrapper for Decaffeinate: convert CoffeeScript files to ES6.

        See https://github.com/decaffeinate/decaffeinate for details on the underlying tool.

        Just run the script with no arguments to get started. It will guide you through some steps:
        1.) Create a file called files_to_decaffeinate.txt in the aurelia directory which has the
            path of each file to convert.
        2.) Check the status of each file. Some files may need some additional modification before
            they can be converted.
        3.) Convert each file to be compatible with decaffeinate. An online repl tool can help with
            this process.
        4.) Commit these changes so you have a clean git state.
        5.) Run with the "--convert" option. This creates two git commits to rename the .coffee
            files and run decaffeinate on them.
        6.) Test the changes and get them reviewed, ignoring lint failures.
        7.) Fix the lint failures and any additional style issues in a follow-up commit and send
            that out as a separate review.
    """), formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('-s', '--status', help='(default) check if the files can be converted',
                        action='store_true')
    parser.add_argument('-r', '--repl',
                        help='open the file (specified by index or name) in a browser-based repl')
    parser.add_argument('-d', '--dry-run',
                        help='check if the individual file (specified by index or name) can be converted')
    parser.add_argument('-c', '--convert', help='run the .coffee -> .js conversion',
                        action='store_true')
    parser.add_argument('-p', '--post-decaffeinate-cleanup',
                        help='run post-decaffeinate transforms on the specified JS file')
    args = parser.parse_args()

    if args.post_decaffeinate_cleanup:
        js_path = args.post_decaffeinate_cleanup
        if not js_path.endswith('.js'):
            raise CLIException('Must specify a .js file.')
        base_path = js_path.rsplit('.', 1)[0]
        run_post_decaffeinate_cleanups([base_path])
        return

    basenames = get_base_paths()
    if not basenames:
        welcome_message()
    elif args.dry_run is not None:
        dry_run_file(resolve_file_ref(basenames, args.dry_run))
    elif args.repl is not None:
        open_in_repl(resolve_file_ref(basenames, args.repl))
    elif args.convert:
        convert_files(basenames)
    else:
        if not args.status:
            print_message('Defaulting to --status')
        status(basenames)


def get_base_paths():
    """Get the files to operate on, or the empty array if no files are specified."""
    path = './files_to_decaffeinate.txt'
    if not os.path.isfile(path):
        return []

    with open(path, 'r') as f:
        # Ignore leading and trailing whitespace and blank lines.
        coffee_paths = [p.strip() for p in f.readlines() if p.strip()]

    for coffee_path in coffee_paths:
        if not coffee_path.endswith('.coffee'):
            raise CLIException('Files must have the .coffee extension.')

    base_paths = [coffee_path.rsplit('.', 1)[0] for coffee_path in coffee_paths]
    for base_path in base_paths:
        if not base_path or base_path.startswith('#'):
            continue
        coffee_name = base_path + '.coffee'
        if not os.path.exists(coffee_name):
            raise CLIException('File not found: {}'.format(coffee_name))
        js_name = base_path + '.js'
        if os.path.exists(js_name):
            response = raw_input('JS file {} already exists! Ok to delete? (Y/n)'.format(js_name))
            if response.lower().startswith('n'):
                raise CLIException('Quitting')
            run_command('rm {}'.format(js_name))

    path_counts = Counter(base_paths)
    path_counts.subtract(set(base_paths))
    duplicates = set(path_counts.elements())
    if duplicates:
        raise CLIException(
            'The following base paths were specified more than once: {}'.format(list(duplicates)))

    return base_paths


def welcome_message():
    print_message("""
        Welcome! To get started, create a file called files_to_decaffeinate.txt
        in the aurelia directory. Each line in that file should have an absolute
        or relative path to a .coffee file to convert.

        Once you do that, you can run this script again without any arguments to
        check for any possible problems converting those files.

        You can run this script with --help for a full overview of the process.
    """)


def status(basenames):
    if not check_and_print_status(basenames):
        return
    print_message("""
        All sanity checks passed. You can convert these files by running the following command:
        scripts/dev/decaffeinate.py --convert

        If all goes well, this will generate two git commits to convert these files to .js.
    """)


def check_and_print_status(basenames):
    """Checks whether decaffeinate can be run on all files.

    Returns True if so and False otherwise. Prints all relevant errors out to stdout.
    """
    print_message('Checking decaffeinate on all files.')
    results = []
    for i, basename in enumerate(basenames):
        print_message('Checking file {} of {}'.format(i + 1, len(basenames)))
        results.append(get_decaffeinate_result(basename))

    print_message("""


        Result summary:
        ===============
    """)
    failures = []
    for i, result in enumerate(results):
        if result.error_code is not None:
            failures.append(i)
        prefix = '[OK]   ' if result.error_code is None else '[ERROR:{}]'.format(result.error_code)
        print_message('{}. {} {}'.format(i, prefix, result.filename))

    if failures:
        print_message("""
        Some files are not compatible with decaffeinate! To fix these, open the files in the
        browser-based decaffeinate repl, using the --repl command and the file name or number.
        From there, tweak the code on the left until the conversion works and displays JavaScript
        on the right. Then, copy the code on the left into a local editor and commit the changes.

        Here are the commands to run for the failed files:
        {}
        """.format('\n        '.join(
            'scripts/dev/decaffeinate.py --repl {}'.format(find_best_ref(basenames, num))
            for num in failures)))
        return False

    print_message('All files are compatible with decaffeinate.')

    if not is_git_worktree_clean():
        print_message("""
            You have modifications to your git worktree.
            Please commit any changes before running this command with --convert.
        """)
        return False

    return True


DecaffeinateResult = namedtuple('DecaffeinateResult', ['filename', 'error_code'])


def get_decaffeinate_result(basename):
    coffee_name = basename + '.coffee'
    try:
        result = run_command_allow_failure('./node_modules/.bin/decaffeinate {0}'.format(coffee_name))
        if result.exit_code != 0:
            return DecaffeinateResult(coffee_name, get_error_code(result.stdout))
    finally:
        # Use ":" at the end to ignore exit code (delete if exists, but don't worry if it doesn't).
        run_command('rm {0}.js; :'.format(basename))
    return DecaffeinateResult(coffee_name, None)


def get_error_code(decaffeinate_stdout):
    if "Cannot read property 'name' of undefined" in decaffeinate_stdout:
        return 'name_of_undefined'
    if 'cannot represent Block as an expression' in decaffeinate_stdout:
        return 'block_as_expression'
    if 'cannot find first or last token in String node' in decaffeinate_stdout:
        return 'first_or_last_token'
    if 'expected a colon between the key and expression' in decaffeinate_stdout:
        return 'expected_colon_key_and_expression'
    if "'for own' is not supported yet" in decaffeinate_stdout:
        return 'for_own'
    if "Cannot read property '2' of null" in decaffeinate_stdout:
        return 'cannot_read_2_of_null'
    if 'unmatched }' in decaffeinate_stdout:
        return 'unmatched_close_curly'
    if "'for of' loops used as expressions are not yet supported" in decaffeinate_stdout:
        return 'for_of_expressions'
    if "'for in' loop expressions with non-expression bodies are not supported yet" in decaffeinate_stdout:
        return 'for_in_non_expression_body'
    if 'unexpected indentation' in decaffeinate_stdout:
        return 'unexpected_indentation'
    if 'because it is not editable' in decaffeinate_stdout:
        return 'index_not_editable'
    return 'other'


def is_git_worktree_clean():
    exit_code = os.system('[ -z "$(git status -s)" ]')
    return exit_code == 0


def dry_run_file(basename):
    """Try out decaffeinate on a single file."""
    result = get_decaffeinate_result(basename)
    prefix = '[OK]   ' if result.error_code is None else '[ERROR:{}]'.format(result.error_code)
    print_message('{} {}'.format(prefix, result.filename))


def open_in_repl(basename):
    path = basename + '.coffee'
    with open(path) as f:
        file_contents = f.read()
    full_url = 'http://decaffeinate.github.io/decaffeinate/repl/#?evaluate=true&code={}'.format(
        urllib.quote(file_contents))
    print_message('Opening {} in a browser...'.format(path))
    webbrowser.open(full_url)


def resolve_file_ref(basenames, file_ref):
    """Resolve the named file as either an index or a filename string.

    The file needs to be in files_to_decaffeinate.txt . In the future, we may
    want to also allow absolute paths in general.
    """
    if file_ref.endswith('.coffee'):
        if '/' in file_ref:
            file_ref = file_ref.rsplit('/', 1)[1]
        candidates = [name for name in basenames if (name + '.coffee').endswith('/' + file_ref)]
        if not candidates:
            raise CLIException(
                'Did not find any file in files_to_decaffeinate.txt matching {}'.format(file_ref))
        if len(candidates) > 1:
            raise CLIException(
                'Found multiple files in files_to_decaffeinate.txt matching {}'.format(file_ref))
        return candidates[0]
    else:
        try:
            return basenames[int(file_ref)]
        except (IndexError, ValueError):
            raise CLIException(
                'Invalid file reference {}. Must be either a file name or a number from 0 to {}.'
                .format(file_ref, len(basenames) - 1))


def find_best_ref(basenames, index):
    """Given the base name, return a string that is a valid reference (file name or index)."""
    basename = basenames[index]
    coffee_name = basename + '.coffee'
    file_name = coffee_name.rsplit('/', 1)[1]
    try:
        resolve_file_ref(basenames, file_name)
        return file_name
    except CLIException:
        return str(index)


def convert_files(basenames):
    can_convert = check_and_print_status(basenames)
    if not can_convert:
        # We've already printed a meaningful error message, so just quit.
        return

    def for_all_files(command_format):
        for basename in basenames:
            run_command(command_format.format(basename))

    num_files = len(basenames)
    first_file_short_name = basenames[0].rsplit('/', 1)[1] + '.coffee'

    commit_author = 'Decaffeinate <{}>'.format(get_git_user_email())

    commit_title_1 = 'Decaffeinate: Rename {} and {} other files from .coffee to .js'.format(
        first_file_short_name, num_files - 1)
    commit_title_2 = 'Decaffeinate: Convert {} and {} other files to JS'.format(
        first_file_short_name, num_files - 1)
    commit_title_3 = 'Decaffeinate: Clean up style in {} and {} other files'.format(
        first_file_short_name, num_files - 1)

    for_all_files('cp {0}.coffee {0}.original.coffee')
    for_all_files('git mv {0}.coffee {0}.js')
    run_command('git commit -m "{}" --author "{}"'.format(commit_title_1, commit_author))

    for_all_files('cp {0}.js {0}.coffee')
    for_all_files('./node_modules/.bin/decaffeinate {0}.coffee')
    for_all_files('rm {0}.coffee')
    for_all_files('git add {0}.js')
    run_command('git commit -m "{}" --author "{}"'.format(commit_title_2, commit_author))

    run_post_decaffeinate_cleanups(basenames)
    for_all_files('git add {0}.js')
    run_command('git commit -m "{}" --author "{}"'.format(commit_title_3, commit_author))

    processed_files = ['{}.coffee'.format(basename) for basename in basenames]
    print_message("""
        Done! The following files were processed:
        {processed_files}

        Here's what happened to these files:
        * Each .coffee file was backed up to a .original.coffee file (which is
          ignored through .gitignore).
        * A commit with title "{commit_title_1}" was created.
          It just did a "git mv" on each file to rename .coffee to .js, without
          changing the contents. This is useful to preserve file history.
        * A commit with title "{commit_title_2}" was created.
          It ran the decaffeinate script on each file.
        * A commit with title "{commit_title_3}" was created.
          It ran a number of automated changes to change the generated JavaScript
          code to conform to our coding style, and added a comment at the top of
          the file disabling lint (if necessary) and adding a TODO to do any
          further needed cleanups.

        What you should do now:
        * Run all relevant tests to make sure they still work.
        * Manually test the features touched to make sure they aren't broken.
        * Take a look at the generated JavaScript files and see if you can spot
          any possible _correctness_ issues (but ignore lint/style issues).
          Check the .original.coffee files if you want to refer to the previous
          code.
        * When you think the changes are correct, submit the change for code
          review and land the change in a way that keeps the commits intact.
          Here's an example workflow:

          arc diff HEAD~3  # Submit the last three commits together for code review.
          # ... (Wait for code review).
          arc amend  # Modify the current commit to show that it has been reviewed.
          git fetch  # Make sure origin/dev is up-to-date.
          git checkout origin/dev
          git merge --no-ff [branch name]  # Make a commit merging the change into origin/dev.
          git push origin HEAD:dev  # Push the change.

          But your workflow may be different. Ask for help if you're unsure about
          the git commands to use here.

        * Once that change has landed, you should make another commit that fixes
          any style issues and removes the automatically generated TODO and
          disabled lint.
    """.format(
        processed_files='\n        '.join(processed_files),
        commit_title_1=commit_title_1,
        commit_title_2=commit_title_2,
        commit_title_3=commit_title_3,
    ))


def run_post_decaffeinate_cleanups(basenames):
    jscodeshift_scripts = [
        './scripts/dev/codemods/arrow-function.js',
        './scripts/dev/codemods/rd-to-create-element.js',
        './scripts/dev/codemods/create-element-to-jsx.js',
    ]

    js_filenames = [basename + '.js' for basename in basenames]
    for script in jscodeshift_scripts:
        run_command('./node_modules/.bin/jscodeshift -t {} {}'
                    .format(script, ' '.join(js_filenames)))

    for js_filename in js_filenames:
        if js_filename.endswith('-test.js'):
            prepend_to_file(js_filename, '/* eslint-env mocha */\n')

        eslint_failures = get_eslint_failures(js_filename)

        prepend_text = ''
        if eslint_failures:
            eslint_disable_lines = [
                '/* eslint-disable'
            ] + ['    {},'.format(rule_name) for rule_name in eslint_failures] + [
                '*/'
            ]
            prepend_text += '\n'.join(eslint_disable_lines) + '\n'

        prepend_text += textwrap.dedent("""\
            // TODO: This file was created by scripts/dev/decaffeinate.py .
            // Fix any style issues and re-enable lint.
        """)
        prepend_to_file(js_filename, prepend_text)


def get_eslint_failures(js_filename):
    """Run eslint on the given file and return the names of the failing rules.

    This also passes --fix to eslint, so it modifies the file to fix what issues it can.
    """
    eslint_output_json = run_command_allow_failure(
        './node_modules/.bin/eslint --fix --format json {}'.format(js_filename)).stdout
    eslint_output = json.loads(eslint_output_json)
    rule_ids = [message['ruleId'] for message in eslint_output[0]['messages']]
    return sorted(set(rule_ids))


def prepend_to_file(file_path, prepend_text):
    with open(file_path, 'r') as f:
        contents = f.read()
    new_contents = prepend_text + contents
    with open(file_path, 'w') as f:
        f.write(new_contents)


def get_git_user_email():
    result = run_command_allow_failure('git config user.email')
    return result.stdout.strip()


class CLIException(Exception):
    pass


ProcessResult = namedtuple('ProcessResult', ['stdout', 'exit_code'])


def run_command_allow_failure(command):
    """Run the given command and return stdout regardless of the exit code."""
    print_message('Running {}'.format(command))
    try:
        stdout = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
        print_message('Output was:\n{}'.format(stdout))
        return ProcessResult(stdout, 0)
    except subprocess.CalledProcessError as e:
        print_message('Output was:\n{}'.format(e.output))
        return ProcessResult(e.output, e.returncode)


def run_command(command):
    """Run the given command and fail if the exit code is nonzero."""
    print_message('Running {}'.format(command))
    exit_code = os.system(command)
    if exit_code in [2, 130]:
        raise KeyboardInterrupt()
    if exit_code != 0:
        raise CLIException('Command failed!')


def print_message(message):
    print textwrap.dedent(message)  # noqa


if __name__ == '__main__':
    try:
        main()
    except CLIException as e:
        print_message(e.message)
        sys.exit(1)

license

The MIT License (MIT)

Copyright (c) 2016 Benchling, Inc.

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.