issue-diff.diff

diff --git a/issue-diff.py b/issue-diff.py
index e0de27e..ba53f79 100755
--- a/issue-diff.py
+++ b/issue-diff.py
@@ -1,5 +1,7 @@
 #!/usr/bin/python
 
+from __future__ import print_function
+
 import argparse
 #import bugzilla
 import dateutil.parser
@@ -18,32 +20,106 @@ import osc.core
 
 from osclib.cache import Cache
 
+# Issue summary can contain unicode characters and therefore a string containing
+# either summary or one in which ISSUE_SUMMARY is then placed must be unicode.
+# For example, translation-update-upstream contains bsc#877707 which has a
+# unicode character in its summary.
+BUG_SUMMARY = 'Missing issue references from {project}/{package} to {factory}/{package}'
+BUG_TEMPLATE = u'{message_start}\n\n{issues}'
+MESSAGE_START = 'The following issues were referenced in the changelog for {project}/{package}, but where not found in {factory}/{package} after {newest} days. Review the issues and submit changes to {factory} as necessary.'
+ISSUE_SUMMARY = u'[{label}]({url}) owned by @{owner}: {summary}'
+ISSUE_SUMMARY_PLAIN = '[{label}]({url})'
+
 
-def create_bug(api, project, factory, package, issue, url):
+def create_bug(api, summary, description):
     createinfo = api.build_createbug(
         product='??', # TODO
         version='??',
         component='??',
-        summary='{} missing from {}/{}'.format(issue, factory, package),
-        description='This is an automatically generated bug.\n\n'
-                    'Review {} and if relevant submit to {}.'.format(url, factory))
+        summary=summary,
+        description=description)
 
     return createinfo.id
 
-def issues_get(apiurl, project, package, db, newest, oldest):
+def prompt_continue(change_count):
+    allowed = ['y', 'n', '']
+    if change_count > 0:
+        print('File bug for {} issues and continue? [y/b/s/n/?] (y): '.format(change_count), end='')
+        allowed.append('b')
+        allowed.append('s')
+    else:
+        print('No changes for which to create bug, continue? [y/n] (y): ', end='')
+
+    response = raw_input().lower()
+    if response == '?':
+        print('b = file bug then stop\ns = skip package for now')
+    elif response in allowed:
+        if response == '':
+            response = 'y'
+        return response
+    else:
+        print('Invalid response: {}'.format(response))
+
+    return prompt_continue(change_count)
+
+def prompt_interactive(changes, project, factory, package, newest):
+    with tempfile.NamedTemporaryFile(suffix='.yml') as temp:
+        temp.write(yaml.dump(changes, default_flow_style=False, default_style="'") + '\n')
+        temp.write('# {}/{}\n'.format(project, package))
+        temp.write('# comment or remove lines to whitelist issues')
+        temp.flush()
+
+        editor = os.getenv('EDITOR')
+        if not editor:
+            editor = 'xdg-open'
+        subprocess.call([editor, temp.name])
+
+        changes_after = yaml.safe_load(open(temp.name).read())
+        if changes_after is None:
+            changes_after = {}
+
+        return changes_after
+
+def issue_found(package, label, db):
+    return not(package not in db or db[package] is None or label not in db[package])
+
+def issue_trackers(apiurl):
+    url = osc.core.makeurl(apiurl, ['issue_trackers'])
+    root = ET.parse(osc.core.http_GET(url)).getroot()
+    trackers = {}
+    for tracker in root.findall('issue-tracker'):
+        trackers[tracker.find('name').text] = tracker.find('label').text
+    return trackers
+
+def issue_normalize(trackers, tracker, name):
+    if tracker in trackers:
+        return trackers[tracker].replace('@@@', name)
+    raise Exception('unkown tracker {} for {}'.format(tracker, name))
+
+def issues_get(apiurl, project, package, trackers, db):
     issues = {}
 
     url = osc.core.makeurl(apiurl, ['source', project, package], {'view': 'issues'})
     root = ET.parse(osc.core.http_GET(url)).getroot()
 
     now = datetime.now(tzlocal()) # Much harder than should be.
-    newest = timedelta(days=newest)
-    oldest = timedelta(days=oldest)
     for issue in root.findall('issue'):
-        label = issue.find('label').text
+        # Normalize issues to active API instance issue-tracker definitions.
+        # Assumes the two servers have the name trackers, but different labels.
+        label = issue_normalize(trackers, issue.find('tracker').text, issue.find('name').text)
+
+        # Ignore already processed issues.
         if issue_found(package, label, db):
             continue
 
+        summary = issue.find('summary')
+        if summary is not None:
+            summary = summary.text
+
+        owner = issue.find('owner/login')
+        if owner is not None:
+            owner = owner.text
+
         created = issue.find('created_at')
         updated = issue.find('updated_at')
         if created is not None and created.text is not None:
@@ -51,29 +127,65 @@ def issues_get(apiurl, project, package, db, newest, oldest):
         elif updated is not None and updated.text is not None:
             date = updated.text
         else:
-            # Pre API dating should be safe to ignore.
-            continue
+            # Old date to make logic work.
+            date = '2007-12-12 00:00 GMT+1'
 
         date = dateutil.parser.parse(date)
         delta = now - date
-        if delta >= newest and delta <= oldest:
-            issues[label] = issue.find('url').text
 
-    return issues
+        issues[label] = {
+            'url': issue.find('url').text,
+            'summary': summary,
+            'owner': owner,
+            'age': delta.days,
+        }
 
-def issue_found(package, label, db):
-    return not(package not in db or db[package] is None or label not in db[package])
+    return issues
 
 def package_list(apiurl, project):
-    url = osc.core.makeurl(apiurl, ['search', 'package'], "match=[@project='%s']" % project)
+    url = osc.core.makeurl(apiurl, ['source', project], { 'expand': 1 })
     root = ET.parse(osc.core.http_GET(url)).getroot()
 
     packages = []
-    for package in root.findall('package'):
+    for package in root.findall('entry'):
         packages.append(package.get('name'))
 
     return sorted(packages)
 
+def git_clone(url, directory):
+    return_code = subprocess.call(['git', 'clone', url, directory])
+    if return_code != 0:
+        raise Exception('Failed to clone {}'.format(url))
+
+def sync(config_dir, db_dir):
+    cwd = os.getcwd()
+    devnull = open(os.devnull, 'wb')
+
+    git_sync_dir = os.path.join(config_dir, 'git-sync')
+    git_sync_exec = os.path.join(git_sync_dir, 'git-sync')
+    if not os.path.exists(git_sync_dir):
+        os.makedirs(git_sync_dir)
+        git_clone('https://github.com/simonthum/git-sync.git', git_sync_dir)
+    else:
+        os.chdir(git_sync_dir)
+        subprocess.call(['git', 'pull', 'origin', 'master'], stdout=devnull, stderr=devnull)
+
+    if not os.path.exists(db_dir):
+        os.makedirs(db_dir)
+        git_clone('git@github.com:jberry-suse/osc-plugin-factory-issue-db.git', db_dir)
+
+        os.chdir(db_dir)
+        subprocess.call(['git', 'config', '--bool', 'branch.master.sync', 'true'])
+        subprocess.call(['git', 'config', '--bool', 'branch.master.syncNewFiles', 'true'])
+        subprocess.call(['git', 'config', 'branch.master.syncCommitMsg', 'Sync issue-diff.py changes.'])
+
+    os.chdir(db_dir)
+    return_code = subprocess.call([git_sync_exec])
+    if return_code != 0:
+        raise Exception('Failed to sync local db changes.')
+
+    os.chdir(cwd)
+
 def main(args):
     # Store the default apiurl in addition to the overriden url if the
     # option was set and thus overrides the default config value.
@@ -90,97 +202,131 @@ def main(args):
 
     Cache.init()
 
-    if os.path.exists(args.db):
-        db = yaml.safe_load(open(args.db).read())
-    if db is None:
+    db_dir = os.path.join(args.config_dir, 'issue-db')
+    db_file = os.path.join(db_dir, '{}.yml'.format(args.project))
+    sync(args.config_dir, db_dir)
+
+    if os.path.exists(db_file):
+        db = yaml.safe_load(open(db_file).read())
+        if db is None:
+            db = {}
+        else:
+            print('Loaded db file: {}'.format(db_file))
+    else:
         db = {}
 
+    print('Comparing {} against {}'.format(args.project, args.factory))
 
+    trackers = issue_trackers(apiurl)
     packages_project = package_list(apiurl, args.project)
     packages_factory = package_list(apiurl_default, args.factory)
     packages = set(packages_project).intersection(set(packages_factory))
     new = 0
-    db_changes = {}
     for package in packages:
-        issues_project = issues_get(apiurl, args.project, package, db, args.newest, args.oldest)
-        issues_factory = issues_get(apiurl_default, args.factory, package, db, args.newest, args.oldest)
+        issues_project = issues_get(apiurl, args.project, package, trackers, db)
+        issues_factory = issues_get(apiurl_default, args.factory, package, trackers, db)
 
         missing_from_factory = set(issues_project.keys()) - set(issues_factory.keys())
 
+        # Filtering by age must be done after set diff in order to allow for
+        # matches with issues newer than --newest.
+        for label in set(missing_from_factory):
+            if issues_project[label]['age'] < args.newest:
+                missing_from_factory.remove(label)
+
         if len(missing_from_factory) == 0:
             continue
 
         print('{}: {} missing'.format(package, len(missing_from_factory)))
-        db_changes[package] = {}
-        for issue in missing_from_factory:
-            db_changes[package][issue] = issues_project[issue]
 
-        new += 1
-        if new == args.limit:
-            print('stopped at limit')
+        # Generate summaries for issues missing from factory.
+        changes = {}
+        for issue in missing_from_factory:
+            info = issues_project[issue]
+            summary = ISSUE_SUMMARY if info['owner'] is not None else ISSUE_SUMMARY_PLAIN
+            changes[issue] = summary.format(
+                label=issue, url=info['url'], owner=info['owner'], summary=info['summary'])
+
+        # Prompt user to decide which issues to whitelist.
+        #changes_after = prompt_interactive(changes, args.project, args.factory, package, args.newest)
+        changes_after = changes
+
+        # Determine if any real changes (vs typos) and create text issue list.
+        issues = []
+        if len(changes_after) > 0:
+            for issue, summary in changes.items():
+                if issue in changes_after:
+                    issues.append('- ' + summary)
+
+        # Prompt user about how to continue.
+        #response = prompt_continue(len(issues))
+        response = 'y'
+        if response == 'n':
             break
+        if response == 's':
+            continue
 
-    if len(db_changes) == 0:
-        print('no new missing issues')
-        return
-
-    with tempfile.NamedTemporaryFile(suffix='.yml') as temp:
-        temp.write('# issues present in {}, but missing from {}\n'.format(args.factory, args.project))
-        temp.write('# comment or remove lines to whitelist them\n\n')
-        temp.write(yaml.dump(db_changes, default_flow_style=False,  default_style="'"))
-        temp.flush()
-
-        editor = os.getenv('EDITOR')
-        if not editor:
-            editor = 'xdg-open'
-        return_code = subprocess.call([editor, temp.name])
-
-        db_changes_after = yaml.safe_load(open(temp.name).read())
-
-    #bugzilla_api = bugzilla.Bugzilla(args.bugzilla_apiurl)
-    #if not bugzilla_api.logged_in:
-        #print('Bugzilla credentials required to create bugs.')
-        #bugzilla_api.interactive_login()
-
-    for package, issues in db_changes.items():
-        created, whitelisted = 0, 0
-        for issue, url in issues.items():
+        # File a bug if not all issues whitelisted.
+        if len(issues) > 0:
+            summary = BUG_SUMMARY.format(project=args.project, factory=args.factory, package=package)
+            message = BUG_TEMPLATE.format(
+                message_start=MESSAGE_START.format(
+                    project=args.project, factory=args.factory, package=package, newest=args.newest),
+                issues='\n'.join(issues))
+
+            # TODO Lookup bugzilla component.
+            bug_id = '17'
+            #bug_id = bug_create(bugzilla_api, summary, message)
+
+        # Mark changes in db.
+        notified, whitelisted = 0, 0
+        for issue in changes:
             if package not in db:
                 db[package] = {}
 
-            if issue_found(package, issue, db_changes_after):
-                db[package][issue] = '1234'
-                #db[package][issue] = bug_create(bugzilla_api, args.project, args.factory, package, issue, url)
-                created += 1
+            if issue in changes_after:
+                db[package][issue] = bug_id
+                notified += 1
             else:
                 db[package][issue] = 'whitelist'
                 whitelisted += 1
-        print('{}: {} bug(s) created, {} whitelisted'.format(package, created, whitelisted))
 
-    db_dir = os.path.dirname(args.db)
-    if not os.path.exists(db_dir):
-        os.makedirs(db_dir)
+        # Write out changes after each package to avoid loss.
+        with open(db_file, 'w') as outfile:
+            yaml.dump(db, outfile, default_flow_style=False, default_style="'")
+
+        if notified > 0:
+            print('{}: {} notified in bug #{}, {} whitelisted'.format(package, notified, bug_id, whitelisted))
+        else:
+            print('{}: {} whitelisted'.format(package, whitelisted))
+
+        if response == 'b':
+            break
+
+        new += 1
+        if new == args.limit:
+            print('stopped at limit')
+            break
 
-    with open(args.db, 'w') as outfile:
-        yaml.dump(db, outfile, default_flow_style=False, default_style="'")
+    sync(args.config_dir, db_dir)
 
 
 if __name__ == '__main__':
     description = 'Compare packages from a project against factory for differences in referenced issues and ' \
-                  'present changes to allow whitelisting before creating bugzilla entries.'
+                  'present changes to allow whitelisting before creating bugzilla entries. A database is kept ' \
+                  'of previously handled issues to avoid repeats and kept in sync via a git repository.'
     parser = argparse.ArgumentParser(description=description)
     parser.add_argument('-A', '--apiurl', default='https://api.suse.de', metavar='URL', help='OBS instance API URL')
     parser.add_argument('--bugzilla-apiurl', default='apibugzilla.suse.com', metavar='URL', help='bugzilla API URL')
-    parser.add_argument('-d', '--debug', action='store_true', help='print info useful for debuging')
-    parser.add_argument('-f', '--factory', default='openSUSE:Factory', metavar='PROJECT', help='factory projec to use as baseline for comparison')
+    parser.add_argument('-d', '--debug', action='store_true', help='print info useful for debugging')
+    parser.add_argument('-f', '--factory', default='openSUSE:Factory', metavar='PROJECT', help='factory project to use as baseline for comparison')
     parser.add_argument('-p', '--project', default='SUSE:SLE-12-SP2:GA', metavar='PROJECT', help='project to check for issues that have are not found in factory')
     parser.add_argument('--newest', type=int, default='30', metavar='AGE_IN_DAYS', help='newest issues to be considered')
-    parser.add_argument('--oldest', type=int, default='365', metavar='AGE_IN_DAYS', help='oldest issues to be considered')
     parser.add_argument('--limit', type=int, default='0', help='limit number of packages with new issues processed')
-    parser.add_argument('--db', type=argparse.FileType('rw'), metavar='PROJECT', help='project for which to list devel projects')
+    parser.add_argument('--config-dir', help='configuration directory containing git-sync tool and issue db')
     args = parser.parse_args()
 
-    if args.db is None:
-        args.db = default=os.path.expanduser('~/.osc-plugin-factory/issue-whitelist/{}.yml'.format(args.project))
+    if args.config_dir is None:
+        args.config_dir = os.path.expanduser('~/.osc-plugin-factory')
 
     sys.exit(main(args))