JavaCS3/submodule-helper.py

## submodule-helper.py
#!/usr/bin/env python
# Style Guide: https://github.com/google/styleguide/blob/gh-pages/pyguide.md
# Pythonic: https://docs.python-guide.org/writing/style/

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import re
import sys
import string
import argparse
import subprocess

from collections import defaultdict

# __dir__ = os.path.dirname(os.path.realpath(__file__))

COLORS = {
    'black': u'0;30', 'bright gray': u'0;37',
    'blue': u'0;34', 'white': u'1;37',
    'green': u'0;32', 'bright blue': u'1;34',
    'cyan': u'0;36', 'bright green': u'1;32',
    'red': u'0;31', 'bright cyan': u'1;36',
    'purple': u'0;35', 'bright red': u'1;31',
    'yellow': u'0;33', 'bright purple': u'1;35',
    'dark gray': u'1;30', 'bright yellow': u'1;33',
    'magenta': u'0;35', 'bright magenta': u'1;35',
    'normal': u'0', 'bold': u'1',
}


def echo(*objects, **kwargs):
    sep = kwargs.get('sep', ' ')
    end = kwargs.get('end', '\n')
    out = kwargs.get('file', sys.stdout)

    msg = sep.join(map(str, objects)) + end
    color = kwargs.get('color')
    isatty = hasattr(out, 'isatty') and out.isatty()

    if color and isatty:
        out.write(u'\033[%sm%s\033[0m' % (COLORS[color], msg))
    else:
        out.write(msg)

    out.flush()


def bold(msg):
    echo(msg, color='bold')


def ok(msg):
    echo(msg, color='green')


def warn(msg):
    echo(msg, color='yellow')


def error(msg):
    echo(msg, color='red')


def banner(msg, width=30, color='blue'):
    echo('%s\n%s\n%s' % ('=' * width, msg, '=' * width), color=color)


def sh(code, debug=False, check_output=False, **kwargs):
    """Run shell script and wait for it complete

    Warning: this function has a security hazard, see:
    https://docs.python.org/2/library/subprocess.html#frequently-used-arguments

    Example:

    sh('echo hello world && ls')
    """
    flag = 'set -o errexit;set -o pipefail;'

    if debug:
        flag += 'set -o xtrace;'

    func = subprocess.check_output if check_output else subprocess.check_call

    return func(flag + code, shell=True, **kwargs)


def sh_out(code, **kwargs):
    return sh(code, check_output=True, **kwargs).rstrip()


def subst(tpl, **env):
    return string.Template(tpl).safe_substitute(**env)


def cmd(cmds, debug=False, check_output=False, **kwargs):
    """Execute a command and wait for it complete

    Example:

    cmd(['ls', '-a', '-l'])
    """

    if debug:
        echo('cmd: %s' % cmds)

    func = subprocess.check_output if check_output else subprocess.check_call

    return func(cmds, **kwargs)


def cmd_out(cmds, **kwargs):
    return cmd(cmds, check_output=True, **kwargs).rstrip()


GIT = os.environ.get('GIT', 'git')
GIT_SUBMODULE_STATUS_REGEX = re.compile(
    r'(?P<changed>[\s|+])(?P<hash>\w+)\s(?P<path>.*)?\s\((?P<head>.*)\)',
    re.MULTILINE
)


def git(*args, **kwargs):
    return cmd([GIT] + list(args), **kwargs)


def git_out(*args, **kwargs):
    return git(*args, check_output=True, **kwargs).rstrip()


def git_submodule_status(path='', recursive=False):
    cmds = ['submodule', 'status']

    if recursive:
        cmds.append('--recursive')

    if path:
        cmds.append(path)

    status = (
        m.groupdict()
        for m in GIT_SUBMODULE_STATUS_REGEX.finditer(git_out(*cmds))
    )

    def _set_changed(d):
        d['changed'] = (d['changed'] == '+')
        return d

    return map(_set_changed, status)


def probe_git_dir(path, pattern='.git'):
    candidate = os.path.abspath(os.path.join(path, pattern))

    # git submodule .git is a file
    while not (os.path.isdir(candidate) or os.path.isfile(candidate)):
        next_candidate = os.path.abspath(
            os.path.join(candidate, os.pardir, os.pardir, pattern)
        )

        if next_candidate == candidate:  # /.git
            return None
        else:
            candidate = next_candidate

    return candidate


def group_by(key, iterable):
    assert callable(key), 'key must be callable'

    d = defaultdict(list)

    for each in iterable:
        k = key(each)
        d[k].append(each)

    return d


def _sync_submodule_impl(path, recursive, debug, no_push=False, refspec=None):
    status = git_submodule_status(path=path, recursive=recursive)
    status = filter(lambda d: d['changed'], status)

    # we need to group submodule parents because there will be more than one submodule in a repo
    sm_parent_git_dirs = group_by(
        lambda d: probe_git_dir(os.path.join(d['path'], os.pardir)),
        status
    )

    # path depth first order
    sm_parent_git_dirs_dfo = sorted(sm_parent_git_dirs.keys(), reverse=True)

    for sm_parent_git_dir in sm_parent_git_dirs_dfo:
        sm_parent_dir = os.path.abspath(
            os.path.join(sm_parent_git_dir, os.pardir)
        )
        status_list = sm_parent_git_dirs[sm_parent_git_dir]
        submodule_names = map(
            lambda s: os.path.relpath(s['path'], sm_parent_dir),
            status_list
        )

        env = os.environ.copy()
        env['GIT_DIR'] = sm_parent_git_dir

        submodule_name_hashs = map(
            lambda s: (
                os.path.relpath(s['path'], sm_parent_dir),
                s['hash']
            ),
            status_list
        )
        detail_msg = '\n'.join(
            '  {} => {}'.format(name, hash)
            for name, hash in submodule_name_hashs
        )
        commit_msg = 'auto sync submodules: %s\n\n%s' % (
            submodule_names,
            detail_msg
        )

        warn('entering: %s' % sm_parent_dir)
        echo('changed submodules: %s' % submodule_names)
        echo('applying new submodules hash\n%s' % detail_msg)

        git('add', *submodule_names, env=env, debug=debug)
        git('commit', '-m', commit_msg, env=env, debug=debug)

        if no_push:
            warn('skip push')
        else:
            git('push', 'origin', refspec, env=env, debug=debug)

    return sm_parent_git_dirs


def sync_submodule(path='', recursive=True, debug=False, no_push=False, refspec='HEAD:master'):
    count = 0

    if recursive:
        while True:
            changed_git_repos = _sync_submodule_impl(path,
                                                     recursive=True,
                                                     debug=debug,
                                                     no_push=no_push,
                                                     refspec=refspec)
            count += len(changed_git_repos)

            if not changed_git_repos:
                break
    else:
        count += len(_sync_submodule_impl(path,
                                          recursive=False,
                                          debug=debug,
                                          no_push=no_push,
                                          refspec=refspec))

    ok('%d repos synced' % count)


def run_sync(args):
    sync_submodule(path=args.path,
                   recursive=args.recursive,
                   debug=args.verbose,
                   no_push=args.no_push,
                   refspec=args.refspec)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(help='commands')

    sync_parser = subparsers.add_parser('sync', help='sync')
    sync_parser.add_argument('-r', '--recursive', action='store_true')
    sync_parser.add_argument('-v', '--verbose', action='store_true')
    sync_parser.add_argument('--no-push', action='store_true',
                             help='skip to push to remote')
    sync_parser.add_argument('--refspec', default='HEAD:master',
                             help='git remote refspec to push (default: %(default)s)')
    sync_parser.add_argument('path', nargs='?')
    sync_parser.set_defaults(run=run_sync)

    args = parser.parse_args()

    args.run(args)
	#!/usr/bin/env python
	# Style Guide: https://github.com/google/styleguide/blob/gh-pages/pyguide.md
	# Pythonic: https://docs.python-guide.org/writing/style/

	from __future__ import absolute_import
	from __future__ import division
	from __future__ import print_function

	import os
	import re
	import sys
	import string
	import argparse
	import subprocess

	from collections import defaultdict

	# __dir__ = os.path.dirname(os.path.realpath(__file__))

	COLORS = {
	'black': u'0;30', 'bright gray': u'0;37',
	'blue': u'0;34', 'white': u'1;37',
	'green': u'0;32', 'bright blue': u'1;34',
	'cyan': u'0;36', 'bright green': u'1;32',
	'red': u'0;31', 'bright cyan': u'1;36',
	'purple': u'0;35', 'bright red': u'1;31',
	'yellow': u'0;33', 'bright purple': u'1;35',
	'dark gray': u'1;30', 'bright yellow': u'1;33',
	'magenta': u'0;35', 'bright magenta': u'1;35',
	'normal': u'0', 'bold': u'1',
	}


	def echo(objects, *kwargs):
	sep = kwargs.get('sep', ' ')
	end = kwargs.get('end', '\n')
	out = kwargs.get('file', sys.stdout)

	msg = sep.join(map(str, objects)) + end
	color = kwargs.get('color')
	isatty = hasattr(out, 'isatty') and out.isatty()

	if color and isatty:
	out.write(u'\033[%sm%s\033[0m' % (COLORS[color], msg))
	else:
	out.write(msg)

	out.flush()


	def bold(msg):
	echo(msg, color='bold')


	def ok(msg):
	echo(msg, color='green')


	def warn(msg):
	echo(msg, color='yellow')


	def error(msg):
	echo(msg, color='red')


	def banner(msg, width=30, color='blue'):
	echo('%s\n%s\n%s' % ('=' * width, msg, '=' * width), color=color)


	def sh(code, debug=False, check_output=False, **kwargs):
	"""Run shell script and wait for it complete

	Warning: this function has a security hazard, see:
	https://docs.python.org/2/library/subprocess.html#frequently-used-arguments

	Example:

	sh('echo hello world && ls')
	"""
	flag = 'set -o errexit;set -o pipefail;'

	if debug:
	flag += 'set -o xtrace;'

	func = subprocess.check_output if check_output else subprocess.check_call

	return func(flag + code, shell=True, **kwargs)


	def sh_out(code, **kwargs):
	return sh(code, check_output=True, **kwargs).rstrip()


	def subst(tpl, **env):
	return string.Template(tpl).safe_substitute(**env)


	def cmd(cmds, debug=False, check_output=False, **kwargs):
	"""Execute a command and wait for it complete

	Example:

	cmd(['ls', '-a', '-l'])
	"""

	if debug:
	echo('cmd: %s' % cmds)

	func = subprocess.check_output if check_output else subprocess.check_call

	return func(cmds, **kwargs)


	def cmd_out(cmds, **kwargs):
	return cmd(cmds, check_output=True, **kwargs).rstrip()


	GIT = os.environ.get('GIT', 'git')
	GIT_SUBMODULE_STATUS_REGEX = re.compile(
	r'(?P<changed>[\s\|+])(?P<hash>\w+)\s(?P<path>.)?\s\((?P<head>.)\)',
	re.MULTILINE
	)


	def git(args, *kwargs):
	return cmd([GIT] + list(args), **kwargs)


	def git_out(args, *kwargs):
	return git(args, check_output=True, *kwargs).rstrip()


	def git_submodule_status(path='', recursive=False):
	cmds = ['submodule', 'status']

	if recursive:
	cmds.append('--recursive')

	if path:
	cmds.append(path)

	status = (
	m.groupdict()
	for m in GIT_SUBMODULE_STATUS_REGEX.finditer(git_out(*cmds))
	)

	def _set_changed(d):
	d['changed'] = (d['changed'] == '+')
	return d

	return map(_set_changed, status)


	def probe_git_dir(path, pattern='.git'):
	candidate = os.path.abspath(os.path.join(path, pattern))

	# git submodule .git is a file
	while not (os.path.isdir(candidate) or os.path.isfile(candidate)):
	next_candidate = os.path.abspath(
	os.path.join(candidate, os.pardir, os.pardir, pattern)
	)

	if next_candidate == candidate: # /.git
	return None
	else:
	candidate = next_candidate

	return candidate


	def group_by(key, iterable):
	assert callable(key), 'key must be callable'

	d = defaultdict(list)

	for each in iterable:
	k = key(each)
	d[k].append(each)

	return d


	def _sync_submodule_impl(path, recursive, debug, no_push=False, refspec=None):
	status = git_submodule_status(path=path, recursive=recursive)
	status = filter(lambda d: d['changed'], status)

	# we need to group submodule parents because there will be more than one submodule in a repo
	sm_parent_git_dirs = group_by(
	lambda d: probe_git_dir(os.path.join(d['path'], os.pardir)),
	status
	)

	# path depth first order
	sm_parent_git_dirs_dfo = sorted(sm_parent_git_dirs.keys(), reverse=True)

	for sm_parent_git_dir in sm_parent_git_dirs_dfo:
	sm_parent_dir = os.path.abspath(
	os.path.join(sm_parent_git_dir, os.pardir)
	)
	status_list = sm_parent_git_dirs[sm_parent_git_dir]
	submodule_names = map(
	lambda s: os.path.relpath(s['path'], sm_parent_dir),
	status_list
	)

	env = os.environ.copy()
	env['GIT_DIR'] = sm_parent_git_dir

	submodule_name_hashs = map(
	lambda s: (
	os.path.relpath(s['path'], sm_parent_dir),
	s['hash']
	),
	status_list
	)
	detail_msg = '\n'.join(
	' {} => {}'.format(name, hash)
	for name, hash in submodule_name_hashs
	)
	commit_msg = 'auto sync submodules: %s\n\n%s' % (
	submodule_names,
	detail_msg
	)

	warn('entering: %s' % sm_parent_dir)
	echo('changed submodules: %s' % submodule_names)
	echo('applying new submodules hash\n%s' % detail_msg)

	git('add', *submodule_names, env=env, debug=debug)
	git('commit', '-m', commit_msg, env=env, debug=debug)

	if no_push:
	warn('skip push')
	else:
	git('push', 'origin', refspec, env=env, debug=debug)

	return sm_parent_git_dirs


	def sync_submodule(path='', recursive=True, debug=False, no_push=False, refspec='HEAD:master'):
	count = 0

	if recursive:
	while True:
	changed_git_repos = _sync_submodule_impl(path,
	recursive=True,
	debug=debug,
	no_push=no_push,
	refspec=refspec)
	count += len(changed_git_repos)

	if not changed_git_repos:
	break
	else:
	count += len(_sync_submodule_impl(path,
	recursive=False,
	debug=debug,
	no_push=no_push,
	refspec=refspec))

	ok('%d repos synced' % count)


	def run_sync(args):
	sync_submodule(path=args.path,
	recursive=args.recursive,
	debug=args.verbose,
	no_push=args.no_push,
	refspec=args.refspec)


	if __name__ == "__main__":
	parser = argparse.ArgumentParser()
	subparsers = parser.add_subparsers(help='commands')

	sync_parser = subparsers.add_parser('sync', help='sync')
	sync_parser.add_argument('-r', '--recursive', action='store_true')
	sync_parser.add_argument('-v', '--verbose', action='store_true')
	sync_parser.add_argument('--no-push', action='store_true',
	help='skip to push to remote')
	sync_parser.add_argument('--refspec', default='HEAD:master',
	help='git remote refspec to push (default: %(default)s)')
	sync_parser.add_argument('path', nargs='?')
	sync_parser.set_defaults(run=run_sync)

	args = parser.parse_args()

	args.run(args)