patch for ryan harper's merges2csv.py - various new features. see commit message

jgrimm_merges2csv.patch

From 10785fe6d0c658ae45ec44f80b835a2fd8972752 Mon Sep 17 00:00:00 2001
From: Jon Grimm <jon.grimm@canonical.com>
Date: Tue, 19 Jul 2016 12:37:36 -0500
Subject: [PATCH] Add support for single team lookup, outfilename, filtering
 for important.

Converted to GPLv3
PEP8 cleanups
Added support for looking up Responsible teams from package-team-mapping.json
  - Prettify names to something more readable in eventual spreadsheet
Added optional flag '--team <team>' to only show single team output
  - Specifying bad team name, will error out and list known team names
Added optional flag '--exclude-same-upstream' to only show most important needs
Added optional flag '--outfilename <outputfilname>
Added help text via argparse facilities
---
 merges2csv.py | 171 +++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 135 insertions(+), 36 deletions(-)

diff --git a/merges2csv.py b/merges2csv.py
index e036a2d..05dc661 100644
--- a/merges2csv.py
+++ b/merges2csv.py
@@ -1,39 +1,109 @@
-#   Copyright (C) 2013 Canonical Ltd.
+#   Copyright (C) 2015 Canonical Ltd.
 #
 #   Author: Ryan Harper <ryan.harper@canonical.com>
+#   Author: Jon Grimm <jon.grimm@canonical.com>
 #
-#   Curtin is free software: you can redistribute it and/or modify it under
-#   the terms of the GNU Affero General Public License as published by the
-#   Free Software Foundation, either version 3 of the License, or (at your
-#   option) any later version.
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 3 of the License, or
+#   (at your option) any later version.
 #
-#   Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
-#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-#   FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
-#   more details.
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
 #
-#   You should have received a copy of the GNU Affero General Public License
-#   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
-
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software Foundation,
+#   Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import argparse
 import csv
 import json
 import subprocess
 import sys
 import urllib3
 
+KEY_TO_HEADING = {
+    'user': 'Last Uploader',
+    'age': 'Days since last merge',
+    'left_version': 'Ubuntu Version',
+    'base_version': 'Upstream Version',
+    'right_version': 'Debian Version'
+}
+
 
-POCKET = sys.argv[1]
-URL = 'https://merges.ubuntu.com/%s.json' % POCKET
+def heading(key):
+    return KEY_TO_HEADING.get(key, key)
+
+# Prettify team names for output.
+TEAM_TO_NAME = {
+    'kubuntu-bugs': 'kubuntu',
+    'desktop-packages': 'Desktop',
+    'lubuntu-packaging': 'lubuntu',
+    'ubuntuone-hackers': 'UbuntuOne',
+    'ubuntu-sdk-bugs': 'SDK',
+    'snappy-dev': 'Snappy-Dev',
+    'edubuntu-bugs': 'edubuntu',
+    'dx-packages': 'Desktop-DX',
+    'mir-team': 'MIR Team',
+    'ubuntu-security-bugs': 'Security',
+    'translators-packages': 'Translators',
+    'ubuntu-apps-bugs': 'Unity-Apps',
+    'unsubscribed': '',
+    'ubuntu-printing': 'Print',
+    'ubuntu-server': 'Server',
+    'kernel-packages': 'Kernel',
+    'ubuntu-phonedations-bugs': 'Phone',
+    'checkbox-bugs': 'Checkbox',
+    'pkg-ime': 'IME',
+    'ubuntu-openstack': 'OpenStack',
+    'unity-api-bugs': 'Unity-API',
+    'libertine-team': 'Libertine',
+    'unity-ui-bugs': 'Unity-UI',
+    'ubuntu-webapps-bugs': 'WebApps',
+    'xubuntu-bugs': 'xubuntu',
+    'maas-maintainers': 'MAAS',
+    'documentation-packages': 'Docs',
+    'foundations-bugs': 'Foundations',
+}
 
-# download json from merges
+# Parse command line args.
+parser = argparse.ArgumentParser(
+    description="""Helper to convert merges.ubuntu.com to CSV file.""")
+parser.add_argument('COMPONENT', help="""E.g. main, universe, or multiverse""")
+parser.add_argument('--exclude-same-upstream', default=True,
+                    dest='exclude_same',
+                    action='store_true',
+                    help="""Only output packages where Debian upstream
+                    version is greater than the Ubuntu upstream.""")
+parser.add_argument('--team', default=None,
+                    help="""Only output packages
+                    belonging to specificified team,
+                    for example 'ubuntu-server'""")
+parser.add_argument('--outfilename', default=None,
+                    help="""Override the default output filename of
+                    merges-<COMPONENT>.csv""")
+args = parser.parse_args()
+
+URL = 'https://merges.ubuntu.com/%s.json' % args.COMPONENT
+MAP = 'http://people.canonical.com/~ubuntu-archive/package-team-mapping.json'
+
+# Download json from merges
 http = urllib3.PoolManager()
 print('Downloading merge data from ' + URL)
-r = http.request('GET', URL)
-if r.status != 200:
+mergereq = http.request('GET', URL)
+if mergereq.status != 200:
     print('Failed to fetch: ' + URL)
     sys.exit(1)
 
+print('Downloading team package map from ' + MAP)
+mapreq = http.request('GET', MAP)
+if mapreq.status != 200:
+    print('Failed to fetch: ' + MAP)
+    sys.exit(1)
+
+
 def dpkg_compare_versions(upkg, dpkg):
     if "-" in upkg:
         uver = upkg.split("-")[0]
@@ -42,7 +112,7 @@ def dpkg_compare_versions(upkg, dpkg):
         uver = upkg
         dver = dpkg
 
-    result=""
+    result = ""
     if uver == dver:
         result = "="
         return result
@@ -53,21 +123,31 @@ def dpkg_compare_versions(upkg, dpkg):
     else:
         return "<"
 
-KEY_TO_HEADING = {
-    'user': 'Last Uploader',
-    'age' : 'Days since last merge',
-    'left_version': 'Ubuntu Version',
-    'base_version': 'Upstream Version',
-    'right_version': 'Debian Version'
-}
-def heading(key):
-    return KEY_TO_HEADING.get(key, key)
 
-merges = json.loads(r.data.decode('utf-8'))
-#merges = json.loads(open("%s.json" % pocket, "r").read())
+merges = json.loads(mergereq.data.decode('utf-8'))
+team_pkgs = json.loads(mapreq.data.decode('utf-8'))
+
+# List of teams we care about. Currently, all or one team
+if args.team is not None:
+    if args.team in team_pkgs:
+        teams = [args.team]
+    else:
+        # Let's help the poor user with names that we know about.
+        print("Error: The team '{}' does not exist".format(args.team))
+        print("Here are the currently known teams: ")
+        for key, _ in team_pkgs.items():
+            print("  "+key)
+        sys.exit(1)
+else:
+    teams = team_pkgs.keys()
+
+if args.outfilename is not None:
+    outfilename = args.outfilename
+else:
+    outfilename = 'merges-%s.csv' % args.COMPONENT
 
-with open('merges-%s.csv' % POCKET , 'w') as f:
-    writer = csv.writer(f)
+with open(outfilename, 'w') as f:
+    writer = csv.writer(f, lineterminator="\n")
     header_order = [
             heading('source_package'),
             heading('Responsibility'),
@@ -105,22 +185,41 @@ with open('merges-%s.csv' % POCKET , 'w') as f:
     writer.writerow(header_order)
 
     for package in merges:
-        #print('{}:'.format(package['source_package']))
+        skip = False
         dataline = []
+
+        # Walk the CSV keys for special processing.
         for key in key_order:
             if key == 'vs debian':
                 value = dpkg_compare_versions(package['left_version'],
                                               package['right_version'])
+
+                # Limit to only showing where Ubuntu is downlevel at upstream
+                if args.exclude_same:
+                    if value != '<':
+                        skip = True
+
                 # google sheets requires leading tick to indicate field
                 # is not a formula
                 if value == '=':
                     value = "'="
+            elif key == 'responsibility':
+                for team in teams:
+                    if package['source_package'] in team_pkgs[team]:
+                        teamname = TEAM_TO_NAME.get(team, team)
+                        if key in package:
+                            package['responsibility'] += ", " + teamname
+                        else:
+                            package['responsibility'] = teamname
+                        value = package.get(key, '')
+                    else:
+                        skip = True
             else:
                 value = package.get(key, '')
-            #print('  {}: {}'.format(key, value))
+
             dataline.append(value)
 
-        #print(dataline)
-        writer.writerow(dataline)
+        if not skip:
+            writer.writerow(dataline)
 
-print('Wrote merges-%s.csv' % POCKET)
+print('Wrote %s' % outfilename)
-- 
2.8.1