mig.py

#!/usr/bin/env python
'''
Steps:
  1. Create any milestones
  2. Create any labels
  3. Create each issue, linking them to milestones and labels
  3.1: Update status for new issue if closed
  4: Create all the comments for each issue
'''
import getpass
import json
import random
import human_curl as hurl
#hurl.Request.SUPPORTED_METHODS += ('PATCH',)
#didn't work, just use POST

import re
from htmlentitydefs import name2codepoint
def htmlentitydecode(s):
  if s is None: return ''
  s = s.replace(' '*8, '')
  return re.sub('&(%s);' % '|'.join(name2codepoint), 
      lambda m: unichr(name2codepoint[m.group(1)]), s)

from lxml import objectify
from collections import defaultdict

all_text = open(raw_input('Path to xml file: ')).read()
all_xml = objectify.fromstring(all_text)

projects = {}
addtag = raw_input('Make tag for project and add to all[y/n]: ').lower()
addtag = (addtag == 'y')

def add(item):
  try:
    proj = item.project.get('key')
  except AttributeError:
    proj = item.key.text.split('-')[0]

  if proj not in projects:
    projects[proj] = {'Milestones': defaultdict(int), 'Components': defaultdict(int), 'Labels': defaultdict(int), 'Issues': []}

  try:
    resolved_at = ', resolved am ' + item.resolved.text + ''
  except AttributeError:
    resolved_at = ''

  projects[proj]['Issues'].append({"title": item.title.text[item.title.text.index("]") + 2:len(item.title.text)],
    'key': item.key.text,
    "body": htmlentitydecode(item.description.text) + '\n<i>' + item.title.text[0:item.title.text.index("]")+1] + ' ' + item.reporter.get('username') + ' am ' + item.created.text + '' + resolved_at + '</i>',
    "labels": [],
    'open': str(item.status.get('id')) not in ('5','6'),
    'comments': [],
    'duplicates': [],
    'is-duplicated-by': [],
    'relates-to': []
    })

  # github doesn't want you to assign to any name,
  # must be a github user
  #if item.assignee.get('username') != '-1':
  #  projects[proj]['Issues'][-1]['assignee'] = item.assignee.get('username')

  try:
    projects[proj]['Milestones'][item.fixVersion.text] += 1
    # this prop will be deleted later:
    projects[proj]['Issues'][-1]['milestone_name'] = item.fixVersion.text 
  except AttributeError:
    pass
  try:
    projects[proj]['Components'][item.component.text] += 1
    projects[proj]['Issues'][-1]['labels'].append(item.component.text)
  except AttributeError:
    pass
  try:
    for label in item.labels.label:
      projects[proj]['Labels'][label.text] += 1
      projects[proj]['Issues'][-1]['labels'].append(label.text)
  except AttributeError:
    pass
  try:
    for comment in item.comments.comment:
      projects[proj]['Issues'][-1]['comments'].append(htmlentitydecode(comment.text) +
          '\n<i>' +
          comment.get('author') + ' am ' +
          comment.get('created') + '</i>')
  except AttributeError:
    pass
  try:
    for issuelinktype in item.issuelinks.issuelinktype:
      for outwardlink in issuelinktype.outwardlinks:
        for issuelink in outwardlink.issuelink:
          for issuekey in issuelink.issuekey:
            projects[proj]['Issues'][-1][outwardlink.get("description").replace(' ', '-')].append(issuekey.text)
  except AttributeError:
    pass
  try:
    for issuelinktype in item.issuelinks.issuelinktype:
      for inwardlink in issuelinktype.inwardlinks:
        for issuelink in inwardlink.issuelink:
          for issuekey in issuelink.issuekey:
            projects[proj]['Issues'][-1][inwardlink.get("description").replace(' ', '-')].append(issuekey.text)
  except AttributeError:
    pass

for item in all_xml.channel.item:
  add(item)

def prettify(projects):
  def hist(h):
    for key in h.iterkeys():
      print ('%30s(%5d): ' + h[key]*'#') % (key, h[key])
    print

  for proj in projects.iterkeys():
    print proj + ':\n  Milestones:'
    hist(projects[proj]['Milestones'])
    print '  Components:'
    hist(projects[proj]['Components'])
    print '  Labels:'
    hist(projects[proj]['Labels'])
    print

prettify(projects)
print
print 'Components will be combined with labels as github labels...'

proj = raw_input('Project to use: ')
us = raw_input('Repo User: ')
repo = raw_input('Repo to use: ')
user = raw_input('User: ')
pw = getpass.getpass('Pass: ')
url_frag = 'https://api.github.com/repos/' + us + '/' + repo
print 'Making milestones...', url_frag + '/milestones'
print

for mkey in projects[proj]['Milestones'].iterkeys():
  data = {'title': mkey}
  r = hurl.post(url_frag + '/milestones',
      json.dumps(data),
      auth=(user, pw))
  # overwrite histogram data with the actual milestone id now
  if r.status_code == 201:
    content = json.loads(r.content)
    projects[proj]['Milestones'][mkey] = content['number']
    print mkey
  else:
    if r.status_code == 422: # already exists
      ms = json.loads(hurl.get(url_frag + '/milestones?state=open').content)
      ms += json.loads(hurl.get(url_frag + '/milestones?state=closed').content)
      f = False
      for m in ms:
        if m['title'] == mkey:
          projects[proj]['Milestones'][mkey] = m['number']
          print mkey, 'found'
          f = True
          break
      if not f:
        exit('Could not find milestone: ' + mkey)
    else:
      print 'Failure!', r.status_code, r.content

print
print 'Making labels...'
projects[proj]['Components'].update(projects[proj]['Labels'])
#if proj == 'DDB':
#  projects[proj]['Components']['DDB'] = 0
if addtag:
  projects[proj]['Components'][proj] = 0
for lkey in projects[proj]['Components'].iterkeys():
  data = {'name': lkey, 'color': '%.6x' % random.randint(0, 0xffffff)}
  r = hurl.post(url_frag + '/labels',
      json.dumps(data),
      auth=(user, pw))
  if r.status_code == 201:
    print lkey
  else:
    print 'Failure!', r.status_code, r.content

def addcomment(user, pw, url_frag, number, comment):
  r3 = hurl.post(url_frag + '/issues/' + number + '/comments',
    json.dumps({'body': comment}), auth=(user,pw),
    headers={'Accept': 'application/vnd.github.beta.html+json'})
  if r3.status_code == 201:
    print 'Added comment',
  else:
    print 'Failed to add comment!', r3.status_code, r3.content

print
print 'Creating each issue...'
for issue in projects[proj]['Issues']:
  if 'milestone_name' in issue:
    issue['milestone'] = projects[proj]['Milestones'][ issue['milestone_name'] ]
    del issue['milestone_name']
  op = issue['open']
  del issue['open']
  comments = issue['comments']
  del issue['comments']
#  if 'Connectors' in issue['labels'] and 'connectors' not in issue['labels']:
#    issue['labels'].append('connectors')
#  if proj == 'DDB':
#    issue['labels'].append('DDB')
  if addtag:
    issue['labels'].append(proj)

  #screwups:
  #t = issue['title']
  #if proj == 'FNL' and int(re.search('\[[A-Z]+-([0-9]+)\]', t).groups()[0]) >= 56:
  #  print 'skipping ' + t
  #  continue
  #if proj == 'FRG' and int(re.search('\[[A-Z]+-([0-9]+)\]', t).groups()[0]) >= 365:
  #  print 'skipping ' + t
  #  continue
  #if raw_input('skip ' + issue['title'] + '? ').lower() == 'y':
  #  continue
  #if max(t.find('FRG-415'), t.find('FRG-417'), t.find('FRG-418'), t.find('FRG-419'), t.find('FRG-421'), t.find('FRG-423'), t.find('FRG-424'), t.find('FRG-425'), t.find('FRG-427'), t.find('FRG-428'), t.find('FRG-429')) > -1:
  #  print 'skipping ' + t
  #  continue

  print issue['key']
  r = hurl.post(url_frag + '/issues', json.dumps(issue), auth=(user,pw),
      headers={'Accept': 'application/vnd.github.beta.html+json'})

  if r.status_code == 201:
    content = json.loads(r.content)
    print 'Created issue:', issue['title']
    if op == False:
      # this is supposed to be hurl.method.patch() but it fails
      r2 = hurl.post(url_frag + '/issues/' + str(content['number']),
          json.dumps({'state': 'closed'}), auth=(user,pw))
      if r2.status_code == 200:
        print 'Closed issue'
      else:
        print 'Failed to close issue!', r2.status_code, r2.content

    issue['githubid'] = str(content['number'])
    print 'githubid: ' + issue['githubid']

    for comment in comments:
      addcomment(user, pw, url_frag, str(content['number']), comment)


    print
  else:
    print "FFFFFFFFFFFFFFFuuuuuuuuuuuu, couldn't make issue:", issue
    print '*'*10
    print r.status_code, r.content
  print


def resolve_github_id(jiraid):
  print "Resolve " + jiraid + "\n"
  githubid = ''
  for issue in projects[proj]['Issues']:
    if issue['key'] == jiraid:
      githubid = issue['githubid']
      break
  if githubid == '':
    print "Can not resolve " + jiraid + "\n"
  else:
    return "#" + githubid

for issue in projects[proj]['Issues']:
  duplicates = issue['duplicates']
  is_duplicated_by = issue['is-duplicated-by']
  relates_to = issue['relates-to']

  for duplicate_item in duplicates:
    addcomment(user, pw, url_frag, issue['githubid'], "Duplicates: " + resolve_github_id(duplicate_item))

  for is_duplicated_by_item in is_duplicated_by:
    addcomment(user, pw, url_frag, issue['githubid'], "Is duplicated by: " + resolve_github_id(is_duplicated_by_item))

  for relates_to_item in relates_to:
    addcomment(user, pw, url_frag, issue['githubid'], "Relates to: " + resolve_github_id(relates_to_item))