TAVRTGBI (a giant ugly hack)

migrate-bugs.py

#! /usr/bin/env python
# Encoding: UTF-8
from __future__ import unicode_literals

# The Amazing veekun redmine-to-github bug importer!

# (i.e. a giant ugly hack)

# configurable parts are marked KNOB

import os
import re
import urlparse
import urllib
import json
import codecs
import time

import requests
import subprocess
import pprint
from BeautifulSoup import BeautifulSoup
# also needs: pandoc

auth = (
        os.environ.get('GITHUB_USER_EMAIL'),
        os.environ.get('GITHUB_USER_PASSWORD'),
    )

def urljoin(*path):
    while len(path) > 1:
        path = (urlparse.urljoin(path[0] + '/', path[1]), ) + path[2:]
    return path[0]

def redmine_url(*path, **kwargs):
    base = urljoin('http://bugs.veekun.com', *path)
    if kwargs:
        return base + '?' + urllib.urlencode(kwargs)
    else:
        return base

def github_url(*path, **kwargs):
    base = urljoin('https://api.github.com', *path)
    if kwargs:
        return base + '?' + urllib.urlencode(kwargs)
    else:
        return base

def parse_json(*path, **kwargs):
    response = requests.get(redmine_url(redmine_url(*path, **kwargs)))
    return json.loads(response.content.decode('utf-8'))

def yield_redmine_issues():
    issues = parse_json('issues.json', limit=5, status_id='*')
    while True:
        for issue in issues['issues']:
            yield issue
        limit = issues['limit']
        total_count = issues['total_count']
        offset = issues['offset']
        if offset + limit < total_count:
            issues = parse_json('issues.json',
                    status_id='*', limit=100, offset=offset + limit)
        else:
            break


# MAPPING KNOBS HERE
project_map = {
        #'Pokédex library': 'eevee/pokedex',
        #'Spline: Pokédex': 'eevee/spline-pokedex',
        #'Spline': 'eevee/spline',
        #'veekun': 'eevee/veekun',
    }

status_table = {
        1: 'OPEN',
        2: 'ON IT',
        5: 'FIXED',
        7: '7', # http://bugs.veekun.com/issues/424
        8: 'DUPLICATE',
        9: "WON'T FIX",
        10: 'INVALID',
        11: 'NEEDS REVIEW',
    }

project_table = {
        2: 'spline-pokedex',
        4: 'floof',
        5: 'pokedex',
        6: 'porigon-z',
        7: 'spline',
        8: 'veekun',
        9: 'veekun-incoming',
        10: 'spline-users',
        11: 'eefi',
        12: 'spline-forum',
        13: 'spline-gts',
        14: 'spline-front-page',
        15: 'dywypi',
        16: 'kouyou',
        17: 'raidne',
        19: 'sanpera',
        20: 'lurid',
        21: 'conformist',
    }

redmine_to_gh_map = {
        1: 'eevee',
        8: 'zhorken',
        9: 'ootachi',
        11: 'magical',
        16: 'encukou',
        # 18: surskitty
        19: 'sanky',
        30: 'sanky',
        #33: inaki
        35: 'habnabit',
        # 36: _fox
        39: 'epii',
        # 41: Ross Boxfox
        # 42: Sero
        47: 'etesp',
        50: 'etesp',
    }

_users = {}
def get_user(id):
    if id is None:
        return '—'
    try:
        return _users[id]
    except KeyError:
        try:
            _users[id] = parse_json('users/%s.json' % id)['user']['firstname']
        except KeyError:
            _users[id] = '<user %s>' % id
        return _users[id]

_versions = {}
def get_version(id):
    if id is None:
        return '—'
    try:
        return _versions[id]
    except KeyError:
        soup = BeautifulSoup(requests.get(redmine_url('projects/_/versions/%s.xml' % id)).content)
        try:
            _versions[id] = soup.find('h2').text
        except (KeyError, AttributeError):
            _versions[id] = str(id)
        return _versions[id]

get_tracker = {'1': 'Bug', '2': 'Feature', '3': 'Refactor'}.get


def user_link(user_info):
    try:
        gh_name = redmine_to_gh_map[user_info['id']]
    except KeyError:
        return '[{user[name]}](http://bugs.veekun.com/users/{user[id]})'.format(
                user=user_info)
    else:
        return '[{user[name]}](http://github.com/{gh_name})'.format(
                user=user_info,
                gh_name=gh_name,
            )


def tetile_to_markdown(textile):
    markdown, stderr = subprocess.Popen(
        ['pandoc', '--from=textile', '--to=markdown', '--no-wrap', '--email-obfuscation=none'],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE
    ).communicate(textile.encode('utf-8'))
    markdown = markdown.decode('utf-8')
    markdown = re.sub(r'\\$', '', markdown, flags=re.MULTILINE)
    markdown = re.sub(r'^\\#', '\n1.', markdown, flags=re.MULTILINE)
    markdown = re.sub(r'^\\\*', '\n* ', markdown, flags=re.MULTILINE)
    markdown = re.sub(r'^\\>', '\n> ', markdown, flags=re.MULTILINE)
    markdown = re.sub(r'\bcommit:([0-9a-f]{3,40})', r'\1', markdown, flags=re.MULTILINE)
    return markdown

def get_issue_data(issue_number):
    try:
        redmine_issue = parse_json('issues/%s.json' % issue_number,
                include='children,attachments,relations,journals')['issue']
    except ValueError:
        return None

    labels = []

    # Description: Convert from Textile to Markdown
    description = re.sub('\n *', ' ', '''
        **Reported by {user_link}
        on {date}
        ·
        Migrated from [Redmine issue {id}](http://bugs.veekun.com/issues/{id})**
        ''').strip().format(
            id=redmine_issue['id'],
            user_link=user_link(redmine_issue.pop('author')),
            date=redmine_issue.pop('created_on'),
        )
    description += '\n\n---\n\n'
    description += tetile_to_markdown(redmine_issue.pop('description'))
    description += '\n\n'
    description += '\n    Redmine metadata:'
    updated_on = redmine_issue.pop('updated_on')  # stringly typed
    if updated_on:
        description += '\n    Updated on: ' + updated_on
    start_date = redmine_issue.pop('start_date', None)
    if start_date:
        description += '\n    Start date: ' + start_date
    due_date = redmine_issue.pop('due_date', None)
    if due_date:
        description += '\n    Due date: %s' % due_date
    done_ratio = redmine_issue.pop('done_ratio')
    if done_ratio:
        description += '\n    Done ratio: %s%%' % done_ratio
    estimated_hours = redmine_issue.pop('estimated_hours', None)
    if estimated_hours:
        description += '\n    Estimated hours: %s' % estimated_hours
    parent = redmine_issue.pop('parent', None)
    if parent:
        description += '\n    Parent: %s' % parent
    relations = redmine_issue.pop('relations', None)
    if relations:
        description += '\n    Relations:'
        for relation in relations:
            description += '\n        {relation_type} #{issue_id}'.format(**relation)
    subtasks = redmine_issue.pop('children', None)
    if subtasks:
        description += '\n    Subtasks:'
        for subtask in subtasks:
            description += '\n        #{id}: {subject}'.format(**subtask)

    # Priority: make label if not default
    priority = redmine_issue.pop('priority')['name']
    if priority != 'Normal':
        labels.append({
                'Low': 'low-priority',
                'High': 'high-priority',
                'Urgent': 'urgent',
                'Immediate': 'immediate',
            }[priority])

    # Custom fields: Difficulty -- make label if not default
    custom_fields = {}
    for field in redmine_issue.pop('custom_fields'):
        name = field['name']
        value = field['value']
        if name == 'Difficulty':
            if value != 'Normal':
                labels.append(value.lower())

    # Tracker (Bug/fature/refactor): make label if not default
    tracker = redmine_issue.pop('tracker')['name']
    if tracker != 'Bug':
        labels.append(tracker.lower())

    # Status: make label if not default
    status = redmine_issue.pop('status')
    assert status_table[status['id']] == status['name'], status
    if status['name'] not in ('OPEN', 'FIXED'):
        labels.append(status['name'])

    # Category: Make label
    category = redmine_issue.pop('category', None)
    if category:
        labels.append(category['name'].lower())

    fixed_version = redmine_issue.pop('fixed_version', None)
    if fixed_version:
        labels.append('target-version %s' % re.sub(r'[^a-zA-Z0-9 _]', '-', fixed_version['name']))

    labels = [re.sub(r"[^0-9a-zA-Z .]+", '-', l.replace('é', 'e')) for l in labels]

    github_issue = dict(
            title = redmine_issue.pop('subject'),
            body = description,
            labels = labels,
        )

    # Assignee: use a look-up table
    assignee = redmine_issue.pop('assigned_to', None)
    if assignee:
        try:
            github_issue['assignee'] = redmine_to_gh_map[assignee['id']]
        except KeyError:
            labels.append('assigned-to %s' % assignee['name'])

    # Important metadata
    project = redmine_issue.pop('project')
    sanitized_name = project['name'].lower().replace('é', 'e')
    sanitized_name = re.sub('[^a-z]+', '-', sanitized_name)
    if sanitized_name != 'pokedex-library':
        assert project_table[project['id']] == sanitized_name, project
    meta = dict(
            number=redmine_issue.pop('id'),
            project=project['name'],
            status=status['name'],
            open=status['name'] in ('OPEN', 'ON IT', 'NEEDS REVIEW'),
        )

    comments = []
    for journal in sorted(redmine_issue.pop('journals'), key=lambda j: j['created_on']):
        body = re.sub('\n *', ' ', '''**Comment by {user_link} from {date}**''').strip().format(
                user_link=user_link(journal.pop('user')),
                date=journal.pop('created_on'),
            )
        body += '\n\n'
        body += tetile_to_markdown(journal.pop('notes', ''))
        if body:
            # XXX: Spam
            body += '\n\n'
        for detail in journal.pop('details'):
            if detail['property'] == 'attr' and detail['name'] == 'status_id':
                body += '\n    Status: %s ⇨ %s' % (
                        status_table[int(detail['old_value'])],
                        status_table[int(detail['new_value'])])
            elif detail['property'] == 'attr' and detail['name'] == 'project_id':
                body += '\n    Project: %s ⇨ %s' % (
                        project_table[int(detail['old_value'])],
                        project_table[int(detail['new_value'])])
            elif detail['property'] == 'attr' and detail['name'] == 'done_ratio':
                body += '\n    Done: %s%% ⇨ %s%%' % (
                        detail['old_value'],
                        detail['new_value'])
            elif detail['property'] == 'attr' and detail['name'] == 'assigned_to_id':
                body += '\n    Assignee ⇨ %s' % get_user(detail.get('new_value'))
            elif detail['property'] == 'attr' and detail['name'] == 'fixed_version_id':
                body += '\n    Target version: %s ⇨ %s' % (
                        get_version(detail.get('old_value')),
                        get_version(detail.get('new_value')))
            elif detail['property'] == 'attr' and detail['name'] == 'description':
                body += '\n    Updated description'
            elif detail['property'] == 'attr' and detail['name'] == 'subject':
                body += '\n    Updated subject'
            elif detail['property'] == 'attr' and detail['name'] == 'tracker_id':
                body += '\n    Tracker: %s ⇨ %s' % (
                        get_tracker(detail.get('old_value')),
                        get_tracker(detail.get('new_value')))
            elif detail['property'] == 'attr' and detail['name'] in (
                    'priority_id',
                    'category_id',
                    'estimated_hours',
                    'parent_id',
                ):
                body += '\n    %s: %s ⇨ %s' % (
                        detail['name'],
                        detail.get('old_value'),
                        detail.get('new_value'),
                    )
            elif detail['property'] == 'attachment':
                body += '\n    Adds attachment'
            elif detail['property'] == 'cf':
                name = {
                        '2': 'Difficulty',
                    }.get(detail['name'], '<Property %s>' % detail['name'])
                body += '\n    %s: %s' % (
                        name,
                        detail['new_value'],
                    )
            else:
                print meta['number'], detail
                assert 0
        comments.append(body)
        journal.pop('id')  # don't need this
        assert not journal, pprint.pformat(journal)

    for comment in comments:
        github_issue['body'] += '\n\n---\n\n' + comment

    # Be sure that we've handled every scrap of information
    assert not redmine_issue, pprint.pformat(redmine_issue)

    return github_issue, meta

_repo_labels = {}
def ensure_labels_exist(repo, labels):
    try:
        existing = _repo_labels[repo]
    except KeyError:
        existing = _repo_labels[repo] = set()
        url = github_url('repos', repo, 'labels')
        for label in json.loads(requests.get(url, auth=auth).content):
            existing.add(label['name'])
    for label in labels:
        if label not in existing:
            url = github_url('repos', repo, 'labels')
            print url, label
            result = requests.post(url, auth=auth, data=json.dumps(dict(
                    name=label,
                    color='FFFFFF',
                )))
            if not result.ok:
                print result.content
                result.raise_for_status()
            existing.add(label)

def migrate_issue(issue_number, log_file):
    results = get_issue_data(issue_number)
    if results is None:
        log_file.write('#%s: not found\n' % issue_number)
    else:
        github_issue, meta = results
        if not meta['open']:
            log_file.write('#%s closed: %s\n' % (issue_number, meta['status']))
            print meta, 'closed'
            return True
        try:
            repo = project_map[meta['project']]
        except KeyError:
            log_file.write('#%s: project %s not set up\n' % (issue_number, meta['project']))
        else:
            log_file.write('#%s: ' % issue_number)
            try:
                ensure_labels_exist(repo, github_issue['labels'])
                url = github_url('repos', repo, 'issues')
                print url
                result = requests.post(url, auth=auth, data=json.dumps(github_issue))
                if not result.ok:
                    print result.content
                    result.raise_for_status()
                number = json.loads(result.content)['number']
                if not meta['open']:
                    url = github_url('repos', repo, 'issues', str(number))
                    result = requests.patch(url, auth=auth, data=json.dumps(dict(state='closed')))
                    if not result.ok:
                        print result.content
                        result.raise_for_status()
                log_file.write('#%s in %s' % (number, repo))
            except:
                log_file.write('ERROR!')
                raise
            finally:
                log_file.write('\n')
        print github_issue.keys(), meta

def main():
    with codecs.open('migration.log', mode='a', encoding='utf-8') as log_file:
        for i in range(1, 710):  # <--- RANGE KNOB HERE
            if not migrate_issue(i, log_file):
                time.sleep(5)
            log_file.flush()

if __name__ == '__main__':
    main()