crates.py

_example__crates.toml

# This is crates.d/crates.toml

# Master config file for crates in this workspace.
#
# This is a homegrown solution (I'm sorry) to a number of
# annoying problems with using workspaces. Cargo.toml files
# in this workspace are generated from source files that
# contain lines like this:
#
#     !!rsp2-assert-close
#     !!itertools
#
# When you run the following command in the workspace root:
#
#     ./crates gen
#
# Cargo.toml files are created with those lines expanded to e.g.
#
#     rsp2-assert-close = { path = "../../util/assert-close" }
#     itertools = "0.7"
#
# based on metadata in this file.

#--------------------------------------------
# Which crate represents the workspace?
root = "rsp2"

#--------------------------------------------
# Specification of all crates in the workspace and their
# paths relative to the root. The 'crates' script uses this
# information to generate the necessary relative paths
# to include when one crate in the workspace depends on
# another.
[crates]
rsp2                = "."
rsp2-lammps-wrap    = "src/io/lammps"
rsp2-tasks          = "src/tasks"
rsp2-minimize       = "src/minimize"
rsp2-assert-close   = "src/util/assert-close"
rsp2-structure      = "src/structure"
rsp2-structure-io   = "src/io/structure"
rsp2-slice-math     = "src/util/slice-math"
rsp2-util-macros    = "src/util/macros"
rsp2-fs-util        = "src/util/fs"
rsp2-clap           = "src/util/clap"
rsp2-tasks-config   = "src/tasks/config"
rsp2-array-types    = "src/util/array-types"
rsp2-linalg         = "src/linalg"
rsp2-integration-test = "src/util/integration-test"
rsp2-soa-ops        = "src/util/soa-ops"
rsp2-newtype-indices = "src/util/newtype-indices"
rsp2-python         = "src/python"
rsp2-potentials     = "src/potentials"
rsp2-dftbplus       = "src/io/dftbplus"
rsp2c-unfold        = "scripts/unfold_lib"
rsp2-phonopy-io = "src/io/phonopy"
rsp2-sparse = "src/util/sparse"
rsp2-dynmat = "src/dynmat"
#!!INSERTION POINT FOR NEW CRATES!!#
# (do not edit the above line!)

#--------------------------------------------
# Versions for external dependencies, so that we don't have
# to look a whole bunch of them up every single time we make
# a new crate in the workspace.
#
# This also makes it easier to migrate the entire workspace
# to a new version of a dependency.
[preferred-versions]
num-traits = "0.2.3"
num-complex = "0.2.1"
rand = "0.3"
serde = { version = "1.0.91", features = ["rc"] }
slice-of-array = "0.2.1"
# (...many, many entries omitted!)

#--------------------------------------------
# Always optimize some crates when they appear as dependencies.
#
# Choose crates that contain monomorphized instances of code that is critical
# to efficiency. (`pub` generic functions will generally not benefit from the
# feature).
#
# This depends on a cargo nightly feature, so support must be explicitly
# enabled for it in .crates.d/config.toml
[always-optimize]
rsp2-array-types = 3

_example__rsp2-array-types.Cargo.toml

# This is crates.d/rsp2-array-types.Cargo.toml

[package]
!!CRATE-NAME-LINE
version = "0.1.0"
authors = ["Michael Lamparski <diagonaldevice@gmail.com>"]
description = "Provides mathematical vector and matrix types used throughout rsp2."
!!LICENSE-PERMISSIVE
!!DEFAULT-EDITION

[lib]
path = "lib.rs"

[dependencies]
!!slice-of-array
!!num-traits
!!serde { optional = true, features = ["derive"] }
!!rand
!!num-complex

[dev-dependencies]
!!rsp2-assert-close

[features]
default = []

!!NIGHTLY-FEATURE-LINE

# FIXME Once namespaced-features are stabilized, this feature will be renamed to "serde".
# see https://github.com/rust-lang/cargo/issues/1286 (problem and proposal)
#     https://github.com/rust-lang/cargo/issues/5565 (tracking issue, of sorts)
serde-support = ["serde"]

crates.py

#!/usr/bin/env python3
# pylint: disable=cell-var-from-loop

# This file is licensed under WTFPL 2.0.
# I honestly do not care what you do with it.

from contextlib import contextmanager
from pathlib import Path
from argparse import ArgumentParser
import os
import sys
import warnings
import string

import pytoml as toml
import typing as tp

# See crates.d/crates.toml for what this script is all about.
#
# For TOML file modification, this script relies mostly on plain
# text searching and substitution; this is partly because I doubt
# there's any good round-trip parsers for TOML in python,
# and I hate to destroy comments and other fine details.

CARGO_TOMLS_DIR = Path('crates.d')
MASTER_CFG_PATH = Path('crates.d/crates.toml')
MASTER_USER_CFG_PATH = Path('crates.d/config.toml')
MASTER_USER_CFG_EXAMPLE_PATH = Path('crates.d/config.example.toml')
GIT_HOOKS_DIR = Path('crates.d/git-hooks')
# something long enough for people to grep and find this script
INSERTION_POINT_TEXT = "#!!INSERTION POINT FOR NEW CRATES!!#"

def main():
    parser = ArgumentParser(description='manage Cargo.tomls in sp2')

    class no_command:
        @staticmethod
        def __init__(*_junk): pass
        @staticmethod
        def run(_args, fail): fail("no subcommand specified")

    parser.set_defaults(klass=no_command)

    subs = parser.add_subparsers(title='subcommands')

    for klass in ALL_COMMANDS:
        sub = subs.add_parser(klass.cmd, **klass.parser_kw)
        sub.set_defaults(klass=klass)
        klass.configure(sub)

    args = parser.parse_args()
    args.klass.run(args, fail=parser.error)

ALL_COMMANDS = []
def register(klass):
    ALL_COMMANDS.append(klass)
    return klass

#------------

thru = lambda f: lambda g: lambda *a, **kw: f(g(*a, **kw)) # pylint: disable=E0602

#------------

KIND_SPECIAL = object()
KIND_INTERNAL_DEP = object()
KIND_EXTERNAL_DEP = object()

@register
class Gen:
    cmd = 'gen'
    parser_kw = dict(
        description=f'Regenerate Cargo.tomls from the source files in {CARGO_TOMLS_DIR}',
    )

    @staticmethod
    def configure(sub):
        sub.add_argument(
            '--vcs', action='store_true',
            help='temporarily override some user config settings to avoid '
                 'undue changes from being checked into VCS')

    @staticmethod
    def run(args, fail):
        Gen._do_it(args, fail)

    # several other commands invoke 'gen'.  This is kind of a dumb
    # hack to allow them to do so.
    @staticmethod
    def run_like_function(fail):
        # HACK - get the default arguments as an argparse.Namespace
        p = ArgumentParser()
        Gen.configure(p)
        default_args = p.parse_args([])
        Gen._do_it(default_args, fail)

    @staticmethod
    def _do_it(args, fail):
        with pushd(search_for_root(fail)):
            data = read_master_config(fail)
            user_conf = read_master_user_config(fail)
            crates = data.pop('crates')

            # catch some dumb user errors before we get too committed
            for (name, path) in crates.items():

                if not path.is_dir():
                    fail('[crates."{}"]: "{}": no such directory'.format(name, path))

                src = root_rel_src_toml_path(name)
                if not src.is_file():  # pylint: disable=E1101
                    fail('{}: no such file'.format(src))

            # FIXME vvv this is waaaaaay too application-specific.
            binary_shim_pairs = get_binary_shim_pairs()
            make_binary_shims(binary_shim_pairs)
            # FIXME ^^^ this is waaaaaay too application-specific.

            lookup_get = make_macro_lookup(args.vcs, fail)

            if user_conf.simulated_workspace.enabled:
                simulated_workspace_gen_symlinks(crates, user_conf.simulated_workspace, fail)

            # generate modified forms of each Cargo.toml
            for (name, path) in crates.items():

                src_path = Path(root_rel_src_toml_path(name))
                dest_path = path / 'Cargo.toml'
                with open(src_path) as f:
                    lines = list(f)

                with open(dest_path, 'w') as f:
                    write = lambda *args: print(*args, file=f)
                    write("# DO NOT EDIT THIS FILE BY HAND!!")
                    write("# It has been autogenerated from", src_path)
                    write("#")
                    write("# To update it, run './crates gen' in the workspace root")
                    write()

                    for (i, line) in enumerate(lines):
                        line = line.rstrip('\n')

                        invocation = MacroInvocation.try_parse(line, fail=fail)
                        if invocation is None:
                            write(line)
                            continue

                        macro = lookup_get(invocation.name)
                        if macro is None:
                            s = f"{src_path}:{i}: unknown macro: {format_macro(invocation.name)}"
                            fail(s)

                        write(macro.expand(
                            # Optional inline-TOML supplied after the macro name
                            argument=invocation.argument,
                            # These are the keyword args available to callbacks:
                            # The directory for the Cargo.toml that we are generating.
                            cur_dir=path,
                            # The name of the crate whose Cargo.toml we are generating.
                            cur_name=name,
                            # The name of the macro we are calling
                            macro_name=invocation.name,
                            fail=fail,
                        ))

def make_macro_lookup(vcs_safe, fail):
    with pushd(search_for_root(fail)):
        data = read_master_config(fail)
        crates = data.pop('crates')
        preferred_versions = data.pop('preferred-versions')
        opt_levels = data.pop('always-optimize')
        user_conf = read_master_user_config(fail)
        root_name = get_root_package(fail)

        class MaybeItsAWorkspaceMacro(NoArgMacro):
            def expand_noarg(self, **_ignored):
                # This is unfortunate. The Cargo.toml files should not depend on
                # config, but there is no other way to disable a workspace.
                if user_conf.simulated_workspace.enabled and not vcs_safe:
                    # This message is intended to be seen from a git diff.
                    return '''
# !!! CAUTION !!!
# The workspace table has been removed for the 'simulated-workspace'
# feature of 'crates'. Please do not check this into VCS!!!
#
# To correct it, do the following to temporarily override the
# simulated-workspace setting before committing:
#
#      ./crates gen --vcs
#
#[workspace]
                    '''.strip()
                else:
                    return '''
[workspace]
exclude = ["./scripts"]
                    '''.strip()

        class CrateNameLineMacro(NoArgMacro):
            def expand_noarg(self, *, cur_name, **_ignored):
                return f'name = "{cur_name}"'

        binary_shim_pairs = get_binary_shim_pairs()

        # for defining new macros
        lookup_put = lambda k, v: lookup_d.__setitem__(canonicalize_crate(k), v)
        lookup_get = lambda k: lookup_d.get(canonicalize_crate(k))

        # build a dash-insensitive lookup table of callbacks
        lookup_d = {
            # name = "crate-name"
            'CRATE-NAME-LINE': CrateNameLineMacro(),

            # [workspace].  Or not.
            'MAYBE-ITS-A-WORKSPACE': MaybeItsAWorkspaceMacro(),

            # licenses
            'LICENSE-PERMISSIVE': ConstantMacro('license = "MIT OR Apache 2.0"'),
            'LICENSE-GPL':        ConstantMacro('license = "GPL-3.0"'),

            # Edition
            'DEFAULT-EDITION': ConstantMacro('edition = "2018"'),

            # FIXME this is waaaaaay too application-specific.
            'BINARY-SHIM-BIN-ENTRIES': BinaryShimsMacro(binary_shim_pairs),

            # FIXME this is waaaaaay too application-specific.
            'NIGHTLY-FEATURE-LINE': NightlyFeaturesMacro(lookup_get, crates),

            # FIXME this is waaaaaay too application-specific.
            'MAYBE-CARGO-FEATURES': CargoFeaturesMacro(user_conf, vcs_safe),
            'MAYBE-OPTIMIZE-SOME-DEPS': OptimizeDepsMacro(lookup_get, user_conf, opt_levels, vcs_safe),
        }

        for (name, path) in crates.items():
            lookup_put(name, InternalDepMacro(name, path))
        for (name, rhs) in preferred_versions.items():
            lookup_put(name, ExternalDepMacro(name, rhs))
    return lookup_get

class Macro:
    def expand(self,
               argument: tp.Optional[tp.Union[int, float, str, dict, list]],
               *,
               cur_name: str,
               macro_name: str,
               cur_dir,
               fail):
        raise NotImplementedError

    # A list of the internal dep crate names that would be added if this
    # macro were expanded in the proper location.
    def internal_deps(self):
        return []

    # A list of the external dep crate names that would be added if this
    # macro were expanded in the proper location.
    def external_deps(self):
        return []

class NoArgMacro(Macro):
    def expand(self, argument, *, cur_name, cur_dir, macro_name, fail):
        if argument is not None:
            fail(f'macro {macro_name} takes no arguments!')

        return self.expand_noarg(cur_name=cur_name, cur_dir=cur_dir, macro_name=macro_name, fail=fail)

    def expand_noarg(self, *, cur_name, cur_dir, macro_name, fail):
        raise NotImplementedError

class ConstantMacro(NoArgMacro):
    def __init__(self, value: str):
        self.value = value

    def expand_noarg(self, **_ignored):
        return self.value

class InternalDepMacro(Macro):
    def __init__(self, dep_name: str, dep_dir):
        self.dep_name = dep_name
        self.dep_dir = dep_dir

    def expand(self, argument, *, cur_dir, fail, **_ignored):
        # Don't use Path for this because:
        # - It has no normpath equivalent that doesn't follow links (wtf?)
        # - Its relpath equivalent is too strict
        rel = os.path.normpath(os.path.relpath(self.dep_dir, cur_dir))
        rhs = { "path": rel }

        if argument is not None:
            rhs = extend_dependency_toml(rhs, argument, fail=fail, dep_name=self.dep_name)

        return f'{self.dep_name} = {inline_toml(rhs)}'

    def internal_deps(self):
        return [self.dep_name]

class ExternalDepMacro(Macro):
    def __init__(self, dep_name: str, rhs: tp.Union[dict, str]):
        self.dep_name = dep_name
        self.rhs = rhs

    def expand(self, argument, fail, **_ignored):
        if argument is not None:
            if not isinstance(argument, dict):
                fail("argument to dep macros must be dict, not {repr(argument)}")

            rhs = extend_dependency_toml(self.rhs, argument, fail=fail, dep_name=self.dep_name)
        else:
            rhs = self.rhs
        return f'{self.dep_name} = {inline_toml(rhs)}'

    def external_deps(self):
        return [self.dep_name]

def extend_dependency_toml(dest, src, *, dep_name, fail):
    if isinstance(dest, str):
        if "version" in src:
            that_version = src['version']
            fail(f"two versions specified for {dep_name}: {repr(dest)} and {repr(that_version)}")
        dest = { "version": dest }

    if not isinstance(dest, dict):
        fail(f"Expected a table or version string for {dep_name}, not {repr(dest)}")

    dest = dict(dest)
    for (k, v) in src.items():
        if k == "features":
            dest[k] = sorted(dest.get(k, []) + v)
        else:
            dest[k] = v
    return dest

# a makeshift toml.dumps that uses inline tables
def inline_toml(x):
    if isinstance(x, dict):
        items = ', '.join(f'{k} = {inline_toml(v)}' for (k, v) in x.items())
        return f'{{ {items} }}'
    elif isinstance(x, list):
        items = ', '.join(inline_toml(v) for v in x)
        return f'[{items}]'
    # types that only have an inline representation
    elif isinstance(x, (str, float, int, bool)):
        # I'm not 100% sure if python str() makes a valid toml str
        # so dumb dumb hack dumb hack hack dumb dumb dumb
        s = toml.dumps({'a': x}).strip()
        assert s.startswith('a')
        s = s[1:].strip()
        assert s.startswith('=')
        s = s[1:].strip()
        return s
    else:
        raise TypeError(x)

# dumb hack to make pytoml parse a value written in inline syntax
def parse_inline_toml(s):
    if '\n' in s or '\r' in s:
        raise ValueError(f"More than one line in inline TOML: {repr(s)}")
    return toml.loads(f"a = {s}")['a']

class MacroInvocation:
    def __init__(self, name: str, argument=None):
        self.name = name.replace('_', '-')
        self.argument = argument

    # If a line is a valid macro invocation, parse it.
    # If it is clearly not a macro, return `None`.
    # If it is dubious looking, fail.
    #
    # Intended grammar: (current implementation is sloppy/overly permissive)
    #   [ WHITESPACE ]*
    #   "!!" [ WHITESPACE ]* KEBAB_CASE_IDENTIFIER
    #   [ WHITESPACE [ WHITESPACE ]* INLINE_TOML_VALUE ]?
    #   [ WHITESPACE ]*
    #
    @classmethod
    def try_parse(cls, line, fail) -> tp.Optional["MacroInvocation"]:
        line = line.strip()
        if line[:2] != "!!":
            return None
        line = line[2:]

        parts = line.split(None, 1)
        if len(parts) == 0:
            fail("!! with no macro name")
        elif len(parts) == 1:
            name, = parts
            return cls(name)
        elif len(parts) == 2:
            name, argument = parts
            return cls(name, parse_inline_toml(argument))

        assert False, "unreachable"

def format_macro(name):
    return '!!' + name

# crates are generally compared as dash-insensitive
def canonicalize_crate(crate_name):
    return crate_name.replace('_', '-')

# For simulated workspaces, make all target directories symlinks to
# the root package's target, to help reduce duplicate build
# artefacts even if we are not a true workspace.
#
# Also link together configuration directories for subcrates.
def simulated_workspace_gen_symlinks(crates, conf, fail):
    root_package_name = get_root_package(fail)

    # prepare the root crate first
    if conf.shared_target:
        ensure_not_symlink('target')
        Path('target').mkdir(parents=True, exist_ok=True)

    if conf.shared_dot_idea:
        (Path('.subcrate-cfg') / '.idea').mkdir(parents=True, exist_ok=True) # pylint: disable=E1101

    if conf.shared_dot_vscode:
        (Path('.subcrate-cfg') / '.vscode').mkdir(parents=True, exist_ok=True) # pylint: disable=E1101

    # link the rest
    for (name, path) in crates.items():
        if name != root_package_name:
            if conf.shared_target:
                maybe_ensure_target_link('.', path)

            if conf.shared_dot_idea:
                maybe_ensure_target_link('.subcrate-cfg', path, filename='.idea')

            if conf.shared_dot_vscode:
                maybe_ensure_target_link('.subcrate-cfg', path, filename='.vscode')

# tries to have a crate's target symlink back to the root crate's target,
# but may fail. (no big deal)
#
# The logic here was really intended for just the target directory,
#  but 'name' can be specified to reuse this for files where the logic
#  for target is probably good enough
# (the theme is: "a symlink to a directory that an IDE is very likely
#                 to attempt to regenerate the moment it vanishes")
def maybe_ensure_target_link(root_path, crate_path, filename='target'):
    path = Path(crate_path) / filename
    dest = os.path.relpath(Path(root_path) / filename, crate_path)

    # don't remove existing links unless we have to.
    if path.is_symlink() and readlink(path) == str(dest):
        return

    # As soon as we destroy the existing contents, we are in a race
    #  against the RLS to recreate the directory the way WE want it.
    rm_rf(path)

    # gogogogogogogogo!!!!!!
    try: symlink(dest, path)
    except FileExistsError:
        # Damn, the RLS probably beat us.
        # Let it win; maybe we'll have better luck next time.
        pass

def ensure_not_symlink(path):
    if os.path.islink(path):
        os.unlink(path)

#------------

CARGO_TOML_TEMPLATE = '''
[package]
!!CRATE-NAME-LINE
version = "0.1.0"
authors = ["Michael Lamparski <diagonaldevice@gmail.com>"]
{license}
!!DEFAULT-EDITION

[lib]
path = "lib.rs"

[dependencies]

[features]
!!NIGHTLY-FEATURE-LINE
'''[1:]

LICENSE_COMMENT_PERMISSIVE = '''
/* ************************************************************************ **
** This file is part of rsp2, and is licensed under EITHER the MIT license  **
** or the Apache 2.0 license, at your option.                               **
**                                                                          **
**     http://www.apache.org/licenses/LICENSE-2.0                           **
**     http://opensource.org/licenses/MIT                                   **
**                                                                          **
** Be aware that not all of rsp2 is provided under this permissive license, **
** and that the project as a whole is licensed under the GPL 3.0.           **
** ************************************************************************ */
'''[1:]

LICENSE_COMMENT_GPL = '''
/* ********************************************************************** **
**  This file is part of rsp2.                                            **
**                                                                        **
**  rsp2 is free software: you can redistribute it and/or modify it under **
**  the terms of the GNU General Public License as published by the Free  **
**  Software Foundation, either version 3 of the License, or (at your     **
**  option) any later version.                                            **
**                                                                        **
**      http://www.gnu.org/licenses/                                      **
**                                                                        **
** Do note that, while the whole of rsp2 is licensed under the GPL, many  **
** parts of it are licensed under more permissive terms.                  **
** ********************************************************************** */
'''[1:]

@register
class New:
    cmd = 'new'
    parser_kw = {}

    @staticmethod
    def configure(sub):
        sub.add_argument('--license', required=True, choices=['permissive', 'gpl'])
        sub.add_argument('CRATENAME')
        sub.add_argument('DIR')

    @staticmethod
    def run(args, fail):
        license = args.license

        if not is_root('.'):
            # (I'm too lazy to work out the correct relative paths
            # to be written to crates.toml)
            fail("The 'new' subcommand must be run from the root.")

        name = args.CRATENAME
        path = Path(args.DIR)

        if path.exists():
            fail(f'"{path}": path already exists')
        if (path / root_rel_src_toml_path(name)).exists():
            fail(f'crate "{name}" already exists')

        # Insert a line to crates.toml and the root package Cargo.toml
        # without destroying formatting
        # (do this now for earlier exits on error)
        root_name = get_root_package(fail=fail)
        root_path = root_rel_src_toml_path(root_name)
        with open(root_path) as f:
            root_lines = list(f)
        root_lines = master_config_textual_add(root_path, root_lines, f'!!{name}\n', fail=fail)

        with open(MASTER_CFG_PATH) as f:
            master_lines = list(f)
        master_lines = master_config_textual_add(MASTER_CFG_PATH, master_lines, f'{name} = "{path}"\n', fail=fail)

        # make the directory
        path.mkdir()
        with open(root_rel_src_toml_path(name), 'w') as f:
            toml_license = {
                'permissive': "!!LICENSE-PERMISSIVE",
                'gpl': "!!LICENSE-GPL",
            }[license]
            f.write(CARGO_TOML_TEMPLATE.format(license=toml_license))

        with open(path / 'lib.rs', 'w') as f:
            f.write({
                'permissive': LICENSE_COMMENT_PERMISSIVE,
                'gpl': LICENSE_COMMENT_GPL,
            }[license])

        with open_tmp(root_path) as f:
            f.writelines(root_lines)

        with open_tmp(MASTER_CFG_PATH) as f:
            f.writelines(master_lines)

        # regenerate
        Gen.run_like_function(fail)

#------------

@register
class Mv:
    cmd = 'mv'
    parser_kw = {}

    @staticmethod
    def configure(sub):
        sub.add_argument('CURDIR')
        sub.add_argument('DESTDIR')

    @staticmethod
    def run(args, fail):
        do_mv_or_rm(curdir=args.CURDIR, destdir=args.DESTDIR, fail=fail)

@register
class Rm:
    cmd = 'rm'
    parser_kw = {}

    @staticmethod
    def configure(sub):
        sub.add_argument('DIR')

    @staticmethod
    def run(args, fail):
        do_mv_or_rm(curdir=args.DIR, destdir=None, fail=fail)

# Shared implementation for mv and rm.
# When distdir is None, this deletes.
def do_mv_or_rm(curdir, destdir, fail):
    curdir = Path(curdir)
    if destdir is not None:
        destdir = Path(destdir)

    if not is_root('.'):
        # (I'm too lazy to work out the correct relative paths
        # to be written to crates.toml)
        cmd_name = 'rm' if destdir is None else 'mv'
        fail(f"The '{cmd_name}' subcommand must be run from the root.")
    root = Path('.')

    if destdir is not None and destdir.exists():
        fail(f'"{destdir}": path already exists')
    if not curdir.exists():
        fail(f'"{curdir}": path does not exist')

    crate = crate_name_from_path(curdir)

    # Edit the line in crates.toml without destroying formatting
    # (do this now for earlier exits on error)
    with open(MASTER_CFG_PATH) as f:
        lines = list(f)
    lines = master_config_textual_set_path(lines, crate, destdir, fail=fail)

    if destdir is None: # rm
        rm_rf(str(curdir))
        unlink(root / root_rel_src_toml_path(crate))
    else: # mv
        curdir.rename(destdir)

    with open_tmp(MASTER_CFG_PATH) as f:
        f.writelines(lines)

    # regenerate
    Gen.run_like_function(fail)

#------------

@register
class InstallHooks:
    cmd = 'install-git-hooks'
    parser_kw = {}

    @staticmethod
    def configure(sub):
        pass

    @staticmethod
    def run(args, fail):
        with pushd(search_for_root(fail)):
            nothing_to_do = True
            error = False
            for name in os.listdir(GIT_HOOKS_DIR):
                dest = os.path.join('.git/hooks', name)
                src = os.path.join(GIT_HOOKS_DIR, name)
                dest_rel_src = os.path.relpath(src, start=os.path.dirname(dest))
                if os.path.lexists(dest):
                    if any(x() for x in [
                        (lambda: not os.path.exists(dest)),
                        (lambda: not os.path.exists(src)),
                        (lambda: not os.path.samefile(os.path.realpath(dest), os.path.realpath(src))),
                    ]):
                        warnings.warn(f'Not overwriting {dest}')
                        nothing_to_do = False
                        error = True
                    continue
                else:
                    nothing_to_do = False
                    print(f'Installing {name} hook', file=sys.stderr)
                    os.symlink(dest_rel_src, dest)

            if nothing_to_do:
                print(f'Already up to date!', file=sys.stderr)
            if error:
                sys.exit(1)

@register
class Test:
    cmd = 'test'
    parser_kw = dict(
        description=
            "Replacement for 'cargo test --all' when the 'simulated-workspace'"
            " option is enabled. You won't believe how it works."
    )

    @staticmethod
    def configure(sub):
        pass

    @staticmethod
    def run(_args, fail):
        from subprocess import Popen

        # regenerate
        Gen.run_like_function(fail)

        def do_cargo_test_all(**kw):
            code = Popen(['cargo', 'test', '--all'], **kw).wait()
            if code:
                sys.exit(code)

        with pushd(search_for_root(fail)):
            if read_master_user_config(fail).simulated_workspace.enabled:
                # do the insane thing

                # Create a replica that looks exactly like our project,
                #  except that it truly is a workspace.
                TEST_ROOT = '.secret-test-dir'
                simulated_workspace_ensure_test_replica_exists(TEST_ROOT)
                do_cargo_test_all(cwd=TEST_ROOT)

                # keep TEST_ROOT around for build artefacts
            else:
                # do something reasonable instead.
                do_cargo_test_all()

def simulated_workspace_ensure_test_replica_exists(test_root):
    import shutil
    test_root = Path(test_root)

    if not test_root.exists():
        test_root.mkdir()

    for path in map(Path, ['src', 'tests', 'examples']):
        test_path = test_root / path
        effective_path = test_root / '..' / path
        if effective_path.exists() and not test_path.exists():
            test_path.symlink_to('..' / path)

    test_cargo = test_root / 'Cargo.toml'
    shutil.copyfile('Cargo.toml', test_cargo)
    with open(test_cargo, 'a') as f:
        f.write('\n')
        f.write('[workspace]\n')

#------------

@register
class Dot:
    cmd = 'dot'
    parser_kw = dict(
        description="Show dependency graph."
    )

    @staticmethod
    def configure(sub):
        sub.add_argument('--root', action='store_true', dest='root')
        sub.add_argument('--no-root', action='store_false', dest='root')
        sub.set_defaults(root=False)

        sub.add_argument('--no-leaves', action='store_false', dest='leaves')
        sub.add_argument('--leaves', action='store_true', dest='leaves')
        sub.set_defaults(leaves=True)

    @staticmethod
    def run(args, fail):
        import matplotlib.pyplot as plt
        from networkx.drawing.nx_pydot import write_dot

        # regenerate
        Gen.run_like_function(fail)

        crates = get_internal_crates_and_root_rel_paths(fail)

        g = get_internal_dep_graph(crates, fail)

        if not args.root:
            g.remove_node(get_root_package(fail))

        if not args.leaves:
            leaves = [node for node in g if g.out_degree(node) == 0]
            for node in leaves:
                g.remove_node(node)

        write_dot(g, sys.stdout)
        plt.show()

def get_internal_dep_graph(crates, fail):
    import networkx as nx

    with pushd(search_for_root(fail)):
        g = nx.DiGraph()
        for (crate, path) in crates.items():
            with open(path / 'Cargo.toml') as f:
                d = toml.load(f)
            for dep in d['dependencies']:
                if dep in crates:
                    g.add_edge(crate, dep)
    return g

#------------

@register
class Edit:
    cmd = 'edit'
    parser_kw = dict(
        description="Edit a crate's Cargo.toml."
    )

    @staticmethod
    def configure(sub):
        sub.add_argument('CRATE_OR_PATH', default='.',
            help="Accepts a crate name or a path to a crate directory."
            " You can also do 'edit crates' to edit crates.toml,"
            " or 'edit config' to edit config.toml.")

    @staticmethod
    def run(args, fail):
        import subprocess
        import shutil

        root = search_for_root(fail)

        if args.CRATE_OR_PATH == 'crates':
            toml_path = root / MASTER_CFG_PATH
        elif args.CRATE_OR_PATH == 'config':
            toml_path = root / MASTER_USER_CFG_PATH
            if not toml_path.exists():
                shutil.copyfile(root / MASTER_USER_CFG_EXAMPLE_PATH, toml_path)
                print(f"Created new config file at '{toml_path}'.", file=sys.stderr)
        else:
            if args.CRATE_OR_PATH in get_internal_crate_names(fail):
                crate = args.CRATE_OR_PATH
            else:
                crate = crate_name_from_path(args.CRATE_OR_PATH)
            toml_path = root / root_rel_src_toml_path(crate)

        editor = os.environ.get('EDITOR') or 'vim'
        if subprocess.run([editor, toml_path]).returncode != 0:
            print(f"Failed to edit '{toml_path}'!", file=sys.stderr)
            sys.exit(1)

        # regenerate
        Gen.run_like_function(fail)

#------------

@register
class UnusedDeps:
    cmd = 'unused'
    parser_kw = dict(
        description="List unused external crate dependencies."
    )

    @staticmethod
    def configure(sub):
        pass

    @staticmethod
    def run(_args, fail):

        with pushd(search_for_root(fail)):
            data = read_master_config(fail)
            our_crates = list(data.pop('crates'))
            unused_deps = set(map(canonicalize_crate, data.pop('preferred-versions')))

            for crate in our_crates:
                with open(root_rel_src_toml_path(crate)) as f:
                    lines = list(f)

                to_be_removed = []
                for line in lines:
                    invocation = MacroInvocation.try_parse(line, fail=fail)
                    if invocation is not None:
                        to_be_removed.append(canonicalize_crate(invocation.name))
                unused_deps -= set(to_be_removed)

            for dep in unused_deps:
                print(dep)

#------------

class UserConfig:
    def __init__(self, d):
        d = dict(d)
        simulated_workspace = d.pop('simulated-workspace', {})
        nightly_cargo = d.pop('nightly-cargo', {})
        for key in d:
            warnings.warn(f"Unrecognized user-config section: {key}")

        self.simulated_workspace = CfgSimulatedWorkspace(simulated_workspace)
        self.nightly_cargo = CfgNightlyCargo(nightly_cargo)

    @classmethod
    def default(cls):
        return cls({})

class CfgSimulatedWorkspace:
    def __init__(self, d):
        d = dict(d)
        self.enabled = d.pop('enable', False)
        self.shared_target = d.pop('shared-target', True)
        self.shared_dot_idea = d.pop('shared-dot-idea', True)
        self.shared_dot_vscode = d.pop('shared-dot-vscode', False)
        for key in d:
            warnings.warn(f'Unrecognized user-config key: simulated-workspace.{key}')

class CfgNightlyCargo:
    def __init__(self, d):
        d = dict(d)
        self.always_optimize_some_deps = d.pop('always-optimize-some-deps', False)
        for key in d:
            warnings.warn(f'Unrecognized user-config key: nightly-cargo.{key}')

#------------

def read_master_config(fail):
    root = search_for_root(fail)
    with open(root / MASTER_CFG_PATH) as f:
        d = toml.load(f)
    d['crates'] = {k: Path(v) for (k, v) in d['crates'].items()}
    return d

def read_master_user_config(fail):
    root = search_for_root(fail)
    try:
        with open(root / MASTER_USER_CFG_PATH) as f:
            d = toml.load(f)
        return UserConfig(d)
    except FileNotFoundError:
        return UserConfig.default()

def get_internal_crates_and_root_rel_paths(fail):
    # There used to be logic in here that branched on whether each
    # value in [crates] was a str or a dict... but I think there is
    # already other code that assumes the values are all paths?
    return read_master_config(fail)['crates']

def get_internal_crate_names(fail):
    return set(get_internal_crates_and_root_rel_paths(fail))

def get_root_package(fail):
    return read_master_config(fail)['root']

def crate_name_from_path(path):
    with open(Path(path) / "Cargo.toml") as f:
        return toml.load(f)['package']['name']

def is_root(path):
    return (Path(path) / MASTER_CFG_PATH).exists()

def search_for_root(fail):
    d = Path('.')
    for _ in range(20):
        if is_root(d):
            return d.resolve()
        d = d / '..'
    fail('Cannot find workspace root!')

# set a path for one of our crates, by plaintext search
def master_config_textual_add(path, lines, line, fail):
    lines = list(lines)
    stripped = [x.strip() for x in lines]
    try: i = stripped.index(INSERTION_POINT_TEXT)
    except ValueError:
        fail(f'''
Failed to find the new crate insertion point in '{path}'
It should look like the following:
    {INSERTION_POINT_TEXT}
'''[:-1])
        assert False, "unreachable"

    lines.insert(i, line)
    return lines

# set a path for one of our crates, by plaintext search.
# 'path = None' deletes
def master_config_textual_set_path(lines, name, path, fail):
    lines = list(lines)
    name = canonicalize_crate(name)

    for (i,line) in enumerate(lines):
        if canonicalize_crate(line.strip()).startswith(name):
            # expect exactly two, paired quotes
            quotes = list(find_all_indices_of_any("\"'", line))
            if not (len(quotes) == 2 and line[quotes[0]] == line[quotes[1]]):
                fail(f"""
{MASTER_CFG_PATH}:{i}: This line looks funny, not sure how to edit it.
It should ideally look like:
    {name} = 'current/path/to/{name}'
But at least make sure there are exactly two quotes of the same style.
"""[1:])

            if path is None:
                del lines[i]
            else:
                lines[i] = f'{lines[i][:quotes[0]+1]}{path}{lines[i][quotes[1]:]}'

            return lines
    else: # pylint: disable=useless-else-on-loop
        fail(f"{MASTER_CFG_PATH}: Failed to find line for {name}.")

#------------

# FIXME this is waaaaaay too application-specific.
# Maybe .crates.d could have some simplistic plugin files or something for crap like this.
def get_binary_shim_pairs():
    with open('src/tasks/entry_points.rs') as f:
        lines = list(f)

    identifier_chars = set(''.join([
        string.ascii_lowercase,
        string.ascii_uppercase,
        string.digits,
        '_',
    ]))

    pairs = []
    for cur, nxt in zip(lines, lines[1:]):
        heuristic = "CRATES" # for better debugging
        cur = cur.strip()
        nxt = nxt.strip()
        if heuristic in cur:
            prefix = "// %% CRATES: binary: "
            suffix = " %%"
            assert cur.startswith(prefix)
            assert cur.endswith(suffix)
            bin = cur[len(prefix):-len(suffix)]

            prefix = "pub fn "
            stop   = nxt.index("(")
            assert nxt.startswith(prefix)
            func = nxt[len(prefix):stop].strip()
            assert set(func).issubset(identifier_chars)

            pairs.append((bin, func))

    return sorted(pairs)

def make_binary_shims(pairs):
    shims_dir = Path("src/binary-shims")
    shims_dir.mkdir(exist_ok = True)

    for (bin, func) in pairs:
        with open(shims_dir / f"{bin}.rs", mode='w') as f:
            assert not (set("\\\"") & set(bin))
            print(f"""\
// This file was autogenerated by `crates gen`. Do not edit!
fn main() {{
    let version = rsp2::version::get();
    rsp2_tasks::entry_points::{func}("{bin}", version);
}}
""", end='', file=f)

class BinaryShimsMacro(NoArgMacro):
    def __init__(self, pairs):
        self.pairs = pairs

    def expand_noarg(self, **_ignored):
        entries = []
        for (bin, _func) in self.pairs:
            lines = [
                f"[[bin]]\n",
                f'name = "{bin}"\n',
                f'path = "src/binary-shims/{bin}.rs"\n',
            ]
            entries.append(''.join(lines))
        return '\n'.join(entries)

class NightlyFeaturesMacro(NoArgMacro):
    def __init__(self, lookup_get, crates):
        self.lookup_get = lookup_get
        self.crates = crates

    def expand_noarg(self, *, cur_name, fail, **_ignored):
        deps = read_dep_names_from_macros(cur_name, self.lookup_get, fail=fail, external=False)

        def recursive_feature_line(feature, base_list=()):
            toml_strs = [f'"{f}"' for f in base_list]
            toml_strs += sorted([f'"{d}/{feature}"' for d in deps])
            toml_list = '[' + ', '.join(toml_strs) + ']'
            return f"{feature} = {toml_list}"
        nightly = recursive_feature_line("nightly", ['beta'])
        beta = recursive_feature_line("beta")
        return f'{nightly}\n{beta}'

class CargoFeaturesMacro(NoArgMacro):
    def __init__(self, config, vcs_safe):
        self.features = []
        if not vcs_safe:
            if config.nightly_cargo.always_optimize_some_deps:
                self.features.append('profile-overrides')

    def expand_noarg(self, *_bleh, **_ignored):
        if not self.features:
            return ''

        return f'''
# !!! CAUTION !!!
# Nightly cargo features are enabled in the user config of 'crates'.
# Please do not check this change into VCS!!!
#
# To correct it, do the following to temporarily override the
# cargo-nightly settings before committing:
#
#      ./crates gen --vcs
#

cargo-features = {repr(self.features)}
'''.strip()

class OptimizeDepsMacro(NoArgMacro):
    def __init__(self, lookup_get, config, crate_opt_levels, vcs_safe):
        self.lookup_get = lookup_get
        self.enabled = config.nightly_cargo.always_optimize_some_deps and not vcs_safe
        self.crate_opt_levels = crate_opt_levels

    def expand_noarg(self, *, cur_name, fail, **_ignored):
        if not self.enabled:
            return ''

        deps = read_dep_names_from_macros(cur_name, self.lookup_get, fail=fail)

        out = []
        for dep in deps:
            if dep in self.crate_opt_levels:
                out.append(f'''
[profile.dev.overrides."{dep}"]
opt-level = {self.crate_opt_levels[dep]}
'''.strip())

        return '\n\n'.join(out)

# HACK:
# To figure out the list of crates that a crate depends on during macro
# expansion, we directly re-parse the template Cargo.toml files and look for
# macros that generate deps.
#
# It'd be much nicer to get this information from the expanded TOML,
# but there is a chicken-and-egg problem there as we are currently in
# the middle of macro expansion!
def read_dep_names_from_macros(crate_name, lookup_get, fail, external=True, internal=True):
    deps = []
    with open(root_rel_src_toml_path(crate_name)) as f:
        for line in f:
            invocation = MacroInvocation.try_parse(line, fail=fail)
            if invocation is None:
                continue

            macro = lookup_get(invocation.name)
            if internal: deps.extend(macro.internal_deps())
            if external: deps.extend(macro.external_deps())
    return sorted(deps)

#------------

def root_rel_src_toml_path(crate_name):
    return Path(CARGO_TOMLS_DIR) / f'{crate_name}.Cargo.toml'

# Variant of open(path, 'w') that writes to a temp file,
# then overwrites the requested file only once the context manager
# is exited without an exception
@contextmanager
def open_tmp(path, mode='w'):
    import tempfile
    with tempfile.NamedTemporaryFile(mode=mode, delete=False) as f:
        try: yield f
        except:
            # THIS NEVER HAPPENED
            os.unlink(f.name)
            raise
        f.flush()

    # exiting the 'with' closes (but doesn't delete) the temp file

    # Make the new file official.
    #
    # Writing Cargo.toml seems to cause rebuilds of any crate
    # that has a build.rs, so try to avoid it if we can.
    replace_if_different(path, f.name)

@contextmanager
def pushd(path):
    old = os.path.abspath('.')
    os.chdir(path)
    try: yield None
    finally: os.chdir(old)

def replace_if_different(current, candidate):
    import shutil
    with open(current) as f:
        a = f.read()
    with open(candidate) as f:
        b = f.read()

    if a != b:
        shutil.move(candidate, current)
    else:
        os.unlink(candidate)

def find_all_indices_of_any(needles, haystack):
    needles = set(needles)
    for (i, x) in enumerate(haystack):
        if x in needles:
            yield i

def rm_rf(path):
    import shutil
    path = Path(path)
    try:
        # note: cannot call rmtree on a symlink because lolidunno
        if path.is_dir() and not path.is_symlink():
            shutil.rmtree(path)
        else:
            unlink(path)
    except FileNotFoundError:
        pass

iife = lambda f: f()

#------------

# CLion thinks these functions can't accept pathlib.Path.
def unlink(path):
    return os.unlink(str(path))
def symlink(a, b):
    return os.symlink(str(a), str(b))
def listdir(path):
    return os.listdir(str(path))
def readlink(path):
    return os.readlink(str(path))

#------------

if __name__ == '__main__':
    main()