migrated to https://github.com/vauxoo-dev/gist-vauxoo

bzr2git_script.py

#migrate to "https://github.com/vauxoo-dev/gist-vauxoo"
'''
#!/usr/bin/env python
# coding: utf-8
import sys
import os
import subprocess
import uuid
import ConfigParser
import launchpad

Config = ConfigParser.ConfigParser()
#Config.read("addons_vauxoo.conf")
"""Config file example
[bzr2git_conf]
full_global_path_branches = /Users/moylop260/openerp/bzr2git/oml6
bzr_branches = [("trunk", "lp:~openerp-mexico-maintainer/openerp-mexico-localization/trunk"), ("7.0", "lp:~openerp-mexico-maintainer/openerp-mexico-localization/7.0"),]
"""

current_path = os.path.realpath(os.path.join(os.path.dirname(__file__)))
LP = launchpad.LP()
LP.connect()
CHECK_MP = True

#@argv[1] path with config files
files_cfg_path = sys.argv[1]
print "files_cfg_path",files_cfg_path

lock_file = os.path.join(files_cfg_path, "run.lock")
print "check exists lock_file", lock_file
if os.path.isfile(lock_file):
    print "lock_file exists. Running this instance in other side. Skipped script."
    sys.exit(0)
#TODO: Use threading
#TODO: Use git-bzr revno to check diff of revno's

try:
    print "creating lock_file", lock_file
    open( lock_file, "w" )
    for file_cfg in os.listdir(files_cfg_path):
        if os.path.splitext( file_cfg )[1] in ['.cfg', '.conf', '.config']:
            file_cfg = os.path.join(files_cfg_path, file_cfg)
            print "file_cfg",file_cfg
            if os.path.isfile( file_cfg ):
                print "is file"
                Config.read( file_cfg )
                section = "bzr2git_conf"
                try:
                    Config.options(section)
                    bzr2git = True
                except ConfigParser.NoSectionError:
                    bzr2git = False

                try:
                    full_global_path_branches = Config.get(section, "full_global_path_branches")
                except ConfigParser.NoOptionError:
                    full_global_path_branches = False
                #full_global_path_branches = os.path.join("/Users/moylop260/openerp/bzr2git/repo_local", os.path.basename( full_global_path_branches ) )
                bzr_branches = eval( Config.get(section, "bzr_branches") )
                if CHECK_MP:
                    for (branch_short_name, branch_unique_name) in bzr_branches:
                        mp_data = LP.get_merge_proposals(branch_unique_name)
                        for mp_number in sorted( mp_data.keys(), reverse=True ):#Process last MP first. This is faster with new MP.
                            bzr_branches.append( (branch_short_name + '-MP' + mp_number, 'lp:' + mp_data[mp_number]['name'] ) )

                if bzr2git:
                    if full_global_path_branches:
                        git_repository = os.path.join( full_global_path_branches, "git_repo" )

                        path_bzr_branches = full_global_path_branches

                        for (branch_short_name, branch_unique_name) in bzr_branches:
                            branch_short_name = branch_short_name.replace('trunk', 'master')#git fashion
                            bzr_branch_fullpath = os.path.join( path_bzr_branches, branch_short_name, 'bzr')
                            git_branch_fullpath = os.path.join( path_bzr_branches, branch_short_name, 'git')

                            #old_revno = LP.get_branch_revno(bzr_branch_fullpath) or 0
                            LP.pull_branch(branch_unique_name, bzr_branch_fullpath)
                            #bzr_info = launchpad.get_committer_info(bzr_branch_fullpath)
                            #new_revno = LP.get_branch_revno(branch_unique_name)#Too long time to get only revno
                            bzr_new_revno = LP.get_branch_revno(bzr_branch_fullpath) or 0
                            git_commit_count = launchpad.git_get_commit_count(
                                    git_branch_fullpath) or 0
                            bzr_revno_from_git_commit_count = \
                                 launchpad.bzr_get_revno(git_commit_count,
                                     bzr_branch_fullpath) or 0
                            #bzr_revno_from_git_commit_count = "0"#Comment this line
                            print "bzr_revno_from_git_commit_count,bzr_new_revno",\
                                bzr_revno_from_git_commit_count,bzr_new_revno
                            if bzr_revno_from_git_commit_count <> bzr_new_revno:
                                #TODO: Process date, commiter and msg. Because with bzr uncommit;bzr touch borrar;bzr commit -m "borrar";bzr push --overwrite this will fail.
                                # Because is the same quantity of revno count.
                                bzr_revno_from_git_commit_count = 0#Fix error of only detect last change and delete all other ones :(. TODO: Fix get last update changes
                                LP.bzr2git(bzr_branch_fullpath,
                                    git_branch_path=git_branch_fullpath,
                                    revision=str(bzr_revno_from_git_commit_count)\
                                         + '..' +  str(bzr_new_revno),
                                    git_repo_path=git_repository,
                                    git_branch_name=branch_short_name,
                                )
                                #exit(1)
finally:
    os.unlink(lock_file)
'''

launchpad.py

#migrate to "https://github.com/vauxoo-dev/gist-vauxoo"
'''
from launchpadlib.launchpad import Launchpad, EDGE_SERVICE_ROOT
import os
import threading
import time
import sys
import subprocess

from misc import *

NEW_MERGE_STATUS = ['Needs review', 'Code failed to merge', 'Approved']
MAX_MP = 10

def assert_main_thread():
    t = threading.current_thread()
    #assert t.name == 'MainThread'

class LP(object):
    """
    Expose the few bits we need from launchpadlib.

    Note: any access to launchpad is done in the same thread.
    """

    def __init__(self):
        assert_main_thread()
        self.con = None
        # Mapping team name - last query time to Launchpad
        self.last_lp_call_per_team = {}

    def connect(self):
        assert_main_thread()
        #launchpad = Launchpad.login_anonymously(
        #    'openerp-runbot', 'edge', 'lpcache')
        cachedir = os.path.expanduser("~/.launchpadlib/cache")
        lp_creds = os.path.expanduser("~/.launchpadlib/.lp_creds")
        self.con = Launchpad.login_with(
            'openerp-runbot', EDGE_SERVICE_ROOT, cachedir,
            credentials_file=lp_creds)

    def get_team_branches(self, team_name):
        assert_main_thread()
        try:
            team = self.con.people[team_name]
        except Exception, e: # What's the proper raised exception?
            log("WARNING: no such team:", team_name)
            return []

        default_branch_filter = {'status': ['Experimental', 'Development', 'Mature']}
        last_lp_call = self.last_lp_call_per_team.get(team_name)
        if not last_lp_call:
            team_branches = team.getBranches(**default_branch_filter)
        else:
            team_branches = team.getBranches(
                modified_since=last_lp_call,
                **default_branch_filter)
        self.last_lp_call_per_team[team_name] = time.strftime('%Y-%m-%d')
        return team_branches

    def get_branch(self, unique_name):
        assert_main_thread()
        return self.con.branches.getByUniqueName(unique_name=unique_name)

    def pull_branch(self, unique_name, repo_path):
        #log("branch-update",branch=unique_name)
        #repo_path = b.repo_path + job_suffix
        unique_name = unique_name.startswith("lp:") and \
            unique_name.replace("lp:", "") or unique_name
        if os.path.isdir(os.path.join(repo_path, '.bzr')):
            rc = run(["bzr", "pull", "lp:%s" % unique_name, "--quiet", "--remember", "--overwrite", "-d", repo_path])
        else:
            mkdirs([repo_path])
            rc = run(["bzr", "branch", "--quiet", "--no-tree", "--use-existing-dir", "lp:%s" % unique_name, repo_path ])
        """
        committer_name, committer_xgram, committer_email = \
            get_committer_info(repo_path)
        b.committer_name = committer_name
        b.committer_xgram = committer_xgram
        b.committer_email = committer_email
        log("get-commiter-info", name=committer_name, xgram=committer_xgram, email=committer_email)
        
        log("get-revno-info "+ unique_name)
        revno = get_revno_info( repo_path )
        if revno and b.local_revision_count != revno:
            log("set-revno-info "+ unique_name + " - current revno:" + repr(b.local_revision_count) + " - new revno: " + repr( revno ) )
            b.local_revision_count = revno
        else:
            log("set-revno-info "+ unique_name + " no changes of revno.")
        """
        return True

    def get_merge_proposals(self, unique_name, filters=None, max_mp=None):
        if filters is None:
            filters = {'status': NEW_MERGE_STATUS}
        if max_mp is None:
            max_mp = MAX_MP
        unique_name = unique_name.startswith("lp:") and \
            unique_name.replace("lp:", "") or unique_name
        branch = self.get_branch(unique_name)
        merge_proposals = branch.getMergeProposals(**filters)
        mp_number__mp_obj_dict = dict([(merge_proposal.web_link.split('/')[-1], merge_proposal) for merge_proposal in merge_proposals])
        mp_numbers_sort = sorted( mp_number__mp_obj_dict.keys(), reverse=True )
        mp_data = {}
        for mp_number in mp_numbers_sort[:max_mp]:#last MP
            merge_proposal = mp_number__mp_obj_dict[ mp_number ]
            mp_data[mp_number] = {
                'name': merge_proposal.source_branch.unique_name,
                'mp_obj': merge_proposal,
            }
            #merge_proposal = mp_number__mp_obj_dict[ mp_number ]
            #new_branch = merge_proposal.source_branch.unique_name
        #    print "new_branch",mp_number, new_branch
        #lp.pull_branch(branch.complete_name, branch_path)
        return mp_data

    def get_branch_revno(self, branch_url_path):
        """
        @branch_url_path: String url branch from lp or path branch local folder
        """
        branch_url_path = branch_url_path.startswith("~") and \
            branch_url_path.replace("~", "lp:~") or branch_url_path
        revno = None
        if not 'lp:~' in branch_url_path:
            if not os.path.isdir( os.path.join(branch_url_path, ".bzr") ):
                return revno
        try:
            revno = get_revno_info(branch_url_path)
        except:
            pass
        return revno

    def bzr2git(self, bzr_branch_path, git_branch_path=None, revision=None, git_repo_path=None, git_branch_name=None):
        if os.path.basename(bzr_branch_path.rstrip('/')) == '.bzr':
            bzr_branch_path = os.path.dirname( bzr_branch_path )
        if git_branch_path is None:
            git_branch_path = os.path.join( bzr_branch_path, 'git' )#Without point because make a conflict with .bzr
        git_init(git_branch_path)
        revision_cmd = revision and ["-r", revision] or []
        cmd_args1 = ["bzr", "fast-export", "--plain"]
        cmd_args1.extend( revision_cmd )
        cmd_args1.append( bzr_branch_path )

        cmd_args2 = ["git", \
            "--git-dir=%s"%(git_branch_path),
            "fast-import", "--force"]

        print "run pipe:",' '.join( cmd_args1) + ' | ' + ' '.join( cmd_args2 )
        ps = subprocess.Popen(cmd_args1, stdout=subprocess.PIPE)
        output = subprocess.check_output(cmd_args2,\
                             stdin=ps.stdout)
        ps.wait()
        if git_repo_path and git_branch_name:
            git_init(git_repo_path)
            cmd_args = ["git", "--git-dir=%s"%(git_branch_path), "push", "--force", git_repo_path,\
                "HEAD:" + git_branch_name]
            run(cmd_args)
        return ps

if __name__ == '__main__':
    lp = LP()
    lp.connect()#TODO: Add to a global variable
    branch_test_unique_name = "~vauxoo/addons-vauxoo/6.1"
    mp_data = lp.get_merge_proposals(branch_test_unique_name)
    for mp_number in mp_data.keys():
        print "pull_branch",mp_number, mp_data[mp_number]['name']
        lp.pull_branch( mp_data[mp_number]['name'],
            #os.path.join('/tmp',os.path.basename(mp_data[mp_number]['name']))
            os.path.join('/tmp',os.path.basename(mp_number))
        )

'''

misc.py

#migrated to https://github.com/vauxoo-dev/gist-vauxoo
'''
"""
A few miscellaneous functions (logging, shell command calls, ...).
"""

import fcntl
import logging
import logging.handlers
import re
import signal
import subprocess
import sys
import threading
import time
import os

__all__ = [
    'kill', 'log', 'run', 'run_output', 'get_committer_info', 'underscore', 'dashes', 'has_test_enable_flag', 'has_no_netrpc_flag', 'project_name_from_unique_name',
    'get_revno_info', 'mkdirs', 'git_init', 'git_get_commit_count', 
    'bzr_get_commit_count', 'bzr_get_revno'
    ]

_logger = logging.getLogger('runbot')
_log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s: %(message)s')
#_log_handler = logging.StreamHandler(sys.stdout)
if not os.path.exists('/tmp/run'): 
    os.mkdir('/tmp/run')
_log_handler = logging.handlers.TimedRotatingFileHandler('/tmp/run/runbot.log', 'D', 1, 30)
_log_handler.setFormatter(_log_formatter)
_logger.setLevel(logging.INFO)
_logger.addHandler(_log_handler)

log_lock = threading.Lock()
def log(*l,**kw):
    out = []
    t = threading.current_thread()
    if t.name.startswith('runbot-group-worker-'):
        out.append(t.name[13:])
    elif t.name == 'MainThread':
        out.append('main')
    else:
        out.append(t.name)
    for i in l:
        if not isinstance(i,basestring):
            i=repr(i)
        out.append(i)
    out+=["%s=%r"%(k,v) for k,v in kw.items()]
    with log_lock:
        _logger.info(' '.join(out))

def lock(name):
    fd=os.open(name,os.O_CREAT|os.O_RDWR,0600)
    fcntl.lockf(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)

def nowait():
    signal.signal(signal.SIGCHLD, signal.SIG_IGN)

def run(l,env=None):
    log("run",l)
    env = dict(os.environ, **env) if env else None
    if isinstance(l,list):
        print "run:", ' '.join( l )
        if env:
            rc=os.spawnvpe(os.P_WAIT, l[0], l, env)
        else:
            rc=os.spawnvp(os.P_WAIT, l[0], l)
    elif isinstance(l,str):
        print "run:", l
        tmp=['sh','-c',l]
        if env:
            rc=os.spawnvpe(os.P_WAIT, tmp[0], tmp, env)
        else:
            rc=os.spawnvp(os.P_WAIT, tmp[0], tmp)
    log("run", rc=rc)
    return rc

def kill(pid,sig=signal.SIGKILL):
    try:
        os.kill(pid,sig)
    except OSError:
        pass

"""
def run_pipe(l, cwd=None):
    import pdb;pdb.set_trace()
    lls = []
    ll = []
    for item in l:
        if item == '|':
            lls.append(ll)
            ll = []
            continue
        ll.append(item)
    lls.append(ll)
    #only support one pipe :( TODO: many pipe
    stdins = []
    first_time = True
    for cmd in lls:
        if stdins and stdins[-1]:
            output = subprocess.check_output(cmd, stdin=stdins[-1])
            ps.wait()
        else:
            ps = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            stdins.append( ps.stdout )
    #subps = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=ps.stdout)
#    cmd = lls.pop(0)
#    output = subprocess.check_output(cmd,\
#                         stdin=ps.stdout)
    #ps.wait()
    return lls
"""

#print run_pipe(['hola', '|', 'hola2'])

def run_output(l, cwd=None):
    log("run_output",l)
    print "run output:", ' '.join( l ), "into", cwd
    return subprocess.Popen(l, stdout=subprocess.PIPE, cwd=cwd).communicate()[0]

def mkdirs(dirs):
    if isinstance(dirs, basestring) or isinstance(dirs, str):
        dirs = [dirs]
    for d in dirs:
        if not os.path.exists(d):
            os.makedirs(d)

def git_init(git_path):
    if not os.path.isdir(os.path.join(git_path, 'refs')):
        mkdirs( [git_path] )
        run(["git", "--bare", "init", git_path])
#----------------------------------------------------------
# OpenERP RunBot misc
#----------------------------------------------------------

def underscore(s):
    return s.replace("~","").replace(":","_").replace("/","_").replace(".","_").replace(" ","_")

def dashes(s):
    return s.replace("~","").replace(":","-").replace("/","-").replace(".","-").replace(" ","-").replace("_","-")

def bzr_get_revno(commit_count, branch_path, revno=None):
    if commit_count == 0:
        return '0'
    revnos = bzr_get_all_revno(branch_path, revno)
    revno_from_commit_count = len(revnos) >= commit_count and revnos[commit_count-1] or False
    return revno_from_commit_count

def bzr_get_all_revno(branch_path, revno=None):
    if revno is None:
        revno = "..-1"
    output = run_output(["bzr", "log", "--include-merged", "--line", "-r", str(revno)],
                        cwd=branch_path)
    output_lines = output.strip('\n').split('\n')
    output_lines.reverse()
    revnos = []
    for output_line in output_lines:
        revno_re = re.compile('(?P<revno>[0-9\.]*)?:')
        m = revno_re.match(output_line.strip())
        if m:
            line_revno = m.group('revno').strip()
            try:
                line_revno = int(line_revno)
            except ValueError:
                #Some revno is 1.1.1.1
                pass
            except:
                raise Exception('Error', "Not revno found: [%s]"%(output_line))
            revnos.append(line_revno)
        else:
            raise Exception('Error', "Not revno found: [%s]"%(output_line))
    return revnos

def bzr_get_commit_count(branch_path, revno=None):
    return len( bzr_get_all_revno(branch_path, revno) )

def git_get_commit_count(repo_path, sha=None):
    if sha is None:
        sha = "HEAD"
    revno = False
    if os.path.isdir( os.path.join(repo_path, 'refs') ):
        output = run_output(["git", "--git-dir=%s"%(repo_path), "rev-list", sha, "--count"])
        revno_re = re.compile('(\d+)')
        for i in output.split('\n'):
            m = revno_re.match(i)
            if m:
                revno = int( m.group(1).strip() )
                #revno -= 2#git rev-list make 2 extra commit's diff with bzr revno
                break
    return revno

def get_committer_info(repo_path):
    committer_name = None
    committer_xgram = None
    committer_email = None

    output = run_output(["bzr", "log", "--long", "-r-1"], cwd=repo_path)

    committer_re = re.compile('committer: *(.+)<(.+)@(.+)>')
    for i in output.split('\n'):
        m = committer_re.match(i)
        if m:
            committer_name = m.group(1).strip()
            committer_xgram = m.group(2)
            committer_email = m.group(2) + '@' + m.group(3)
            break

    return committer_name, committer_xgram, committer_email

def get_revno_info(repo_path):
    revno = False
    output = run_output(["bzr", "revno", repo_path])
    revno_re = re.compile('(\d+)')
    for i in output.split('\n'):
        m = revno_re.match(i)
        if m:
            revno = int( m.group(1).strip() )
            break
    return revno

def get_revision_id(repo_path, revno=None):
    if revno is None:
        revno = -1
    output = run_output("bzr", "version-info", "--custom", '--template="{revision_id}\n"', "-r", str(revno), repo_path)

def has_test_enable_flag(server_bin_path):
    """
    Test whether an openerp-server executable has the --test-enable flag.
    (When the flag is present, testing for success/failure is done in a
    different way.)
    """
    p1 = subprocess.Popen([server_bin_path, "--help"],
        stdout=subprocess.PIPE)
    p2 = subprocess.Popen(["grep", "test-enable"], stdin=p1.stdout,
        stdout=subprocess.PIPE)
    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
    output = p2.communicate()[0]
    return output == "    --test-enable       Enable YAML and unit tests.\n"

def has_no_netrpc_flag(server_bin_path):
    """
    Test whether an openerp-server executable has the --no-netrpc flag.
    (When the flag is present, the runbot uses it.)
    """
    p1 = subprocess.Popen([server_bin_path, "--help"],
        stdout=subprocess.PIPE)
    p2 = subprocess.Popen(["grep", "no-netrpc"], stdin=p1.stdout,
        stdout=subprocess.PIPE)
    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
    output = p2.communicate()[0]
    return output == "    --no-netrpc         disable the NETRPC protocol\n"

def project_name_from_unique_name(unique_name):
    m = re.search("/(openobject|openerp)-(addons|server|client-web|web)/", unique_name)
    if m:
        n = m.group(2)
        if n == 'client-web':
            return 'web'
        else:
            return n
    else:
        return None

'''