Ingest a charm into gitlab and perform some common review queue steps

ingest_charm.py

#!/usr/bin/env python
"""
Setup notes:

Git needs to be configured on the machine doing the ingestion.

    git config --global user.name "Administrator"
    git config --global user.email "admin@example.com"

In Gitlab:
- Add the SSH key of the user running ingestion to your master profile:
http://gitlabip/profile/keys

usage, against my fork of the gitlab charm (https://launchpad.net/~aisrael/charms/trusty/reviewqueue/trunk):


./ingest_charm.py charms/trusty/gitlab/

"""


"""
Currently using https://github.com/Itxaka/pyapi-gitlab over
https://github.com/gpocentek/python-gitlab because the former has
some tests and better documentation. I'm not in love with either, though.
"""
import gitlab

import argparse
import tempfile
import os
import os.path
import shutil
import yaml

"""
Use hashlib and time to generate a unique and keep projects distinct
"""
import hashlib
import time

import subprocess
import shlex

# from launchpadlib.launchpad import Launchpad
from charmworldlib.charm import Charm, CharmNotFound

import distutils.dir_util

# from bzrlib import trace
# from bzrlib.bzrdir import BzrDir
from bzrlib.branch import Branch
from bzrlib.plugin import load_plugins
# from bzrlib.repository import Repository

load_plugins()


# TODO: Grab these via argparse?
GITLAB_HOST = "10.0.3.252"
GITLAB_TOKEN = "BvztJyQNVWnnEbyzzSZP"   # The private token from /profile/account


"""
Setup bzr -> git conversion:
cd ~/.bazaar/plugins
bzr branch lp:bzr-fastimport fastimport

"""

def run_command(cmd):
    """
    A temporary method used for quick prototyping, to execute command line
    commands that should be done natively.
    """
    return subprocess.check_output(shlex.split(cmd))


def setup_argparse():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        'charmdir',
        help='The directory containing the charm to review',
        action='store'
    )

    return parser


def make_temp_charm(charmdir):
    """
    Copy the live charm to a temporary directory
    """
    newdir = "%s/%s" % (
        tempfile.mkdtemp(),
        os.path.basename(os.path.normpath(charmdir))
    )
    shutil.copytree(os.path.abspath(charmdir), newdir)

    return newdir

def merge_charms(src, target):
    """
    Merge the src (newer) charm into the target (older) charm
    """
    # src = os.path.abspath(src)
    # target = os.path.abspath("%s/" % target)

    print "Copying %s to %s" % (src, target)
    distutils.dir_util.copy_tree(src, target)
    # shutil.copytree(src, target)

def get_charm_name(charmdir):
    """
    Parse a charm's metadata.yaml and return the value of the name: key
    """
    name = None

    with open('%s/metadata.yaml' % charmdir, 'r') as f:
        metadata = yaml.load(f)
        name = metadata['name']

    return name

def get_charm(name):
    """
    Get the Charm object, or None if not found.
    """
    try:
        charm = Charm(name)
        return charm
    except Exception, e:
        print "No charm found: %s!" % e
        return None

def branch(series='trusty', name=None, charmdir=None):

    if name and charmdir:
        try:
            print "Getting charm %s" % name
            charm = Charm(name)
            # print charm
            print "Opening remote location %s" % charm.code_source['location']
            remote = Branch.open(charm.code_source['location'])

            print "Branching %s to %s" % (charm.code_source['location'], charmdir)
            local = remote.bzrdir.sprout(
                charmdir
            ).open_branch()
            return True
        except Exception, e:
            print "No charm found: %s!" % e
        return False


def main():


    parser = setup_argparse()
    args = parser.parse_args()

    git = gitlab.Gitlab('http://%s' % GITLAB_HOST, token=GITLAB_TOKEN)
    print "Logged in as %s" % git.currentuser()["username"]
    # Copy the charmdir to scratch
    # charmdir = make_temp_charm(args.charmdir)

    # TODO: delete previous cvs artifacts? (.bzr, .svn, .git, etc)
    new_charmdir = os.path.abspath(args.charmdir)
    print "Local charm: %s" % new_charmdir

    # Parse charm metadata to get the charm name
    name = get_charm_name(new_charmdir)
    # print "Charm: %s\n" % name

    """
    This should probably be driven by the RQ UI, so we don't have duplicate
    gitlab projects for an item in the queue.
    """
    # Check to see if the uuid exists; if so, use that one.
    namespace = 'root'
    uuid = hashlib.md5('%s:%s' % (name, time.time())).hexdigest()
    project_id = '%s-%s' % (name, uuid)

    project = git.getproject('%s/%s' % (namespace, project_id))
    if project is False:
        # create new project
        project = git.createproject(
            name=project_id,
        )
    print project

    # Query the charm store to get the latest revision, if it exists
    charm = get_charm(name)
    if charm:
        """
        We have an existing charm in the store,
        so let's build a diff against that.
        """
        charmdir = tempfile.mkdtemp()
        os.chdir(charmdir)

        # TODO: Init the git branch
        run_command("git init")
        run_command('git remote add origin git@%s:%s/%s.git' % (GITLAB_HOST, namespace, project_id))

        if branch(name=name, charmdir=charmdir):
            # Commit changes to git

            run_command("git add .")
            run_command("git commit -am \'Merge trunk\'")
            run_command('git push -u origin master')

            # Commit changes to git so we can see a diff between commits
            # Isn't this kind of like auto-squash, since we don't have all
            # commits between revisions? Feasible to do a fast bzr->git conv?
            # See http://design.canonical.com/2015/01/converting-projects-between-git-and-bazaar/
            # for bzr-fastimport and git fast-import

            # Import charm; copy the local charm into the existing charm

            run_command('git checkout -b %s' % uuid)
            print "Merge %s to %s" % (new_charmdir, charmdir)
            merge_charms(new_charmdir, charmdir)

            #run_command("git commit -m \'\[3daf1b\] Merge proposed\'")
            run_command("git commit -am \'Merge proposed\'")
            run_command('git push -u origin %s' % uuid)

            print "Creating merge request on '%s/%s' for branch %s" % (namespace, project_id, uuid)
            mr = git.createmergerequest(
                project['id'],
                # '%s/%s' % (namespace, project_id), # project to push to
                uuid, # source branch
                'master', # Target branch
                'Something something from the RQ GUI', # Title
            )
            if mr is False:
                print "Merge request failed."
            # else:
            #     print "\nmerge keys:\n"
            #     for key in mr:
            #         print "\n%s=%s\n" % (key, mr[key])


            print "Adding comment to merge request"
            git.addcommenttomergerequest(
                project['id'],
                mr['id'],
                'A general comment goes here'
            )
            
            if git.addcommenttocommit(
                project['id'],
                mr['author'],
                mr['source_branch'],
                'README.md',
                9,
                '"Down with MySQL!"'
            ):
                print "Add comment succeeded!"
            else:
                print "Add comment failed. :("


    # Cleanup
    print "Cleaning up!"
    #shutil.rmtree(charmdir)

if __name__ == "__main__":
    main()