verify_json.py

#! /usr/bin/env python

import argparse
import glob
import json
import os
import sys
import urllib2


def diff(a, b):
    b = set(b)
    return [aa for aa in a if aa not in b]


def analyze_differences(p12n_locales, search_locales, p12n_json):
    ''' Analyze searchplugin differences between the two lists '''
    missing_locales = diff(p12n_locales, search_locales)
    print '\nLocales available in p12n analysis but not in search:',
    # Check if we have browser/aurora for this locale. If we don't, it doesn't
    # have search settings, or ships only in other products.
    for locale in missing_locales[:]:
        if 'aurora' not in p12n_json['locales'][locale]['browser']:
            missing_locales.remove(locale)
    if missing_locales:
        missing_locales.sort()
        print ', '.join(missing_locales)
    else:
        print '(none)'

    added_locales = diff(search_locales, p12n_locales)
    print '\nLocales available in search but not in p12n analysis:',
    if added_locales:
        added_locales.sort()
        print ', '.join(added_locales)
    else:
        print '(none)'


def main():
    # Parse command line options
    parser = argparse.ArgumentParser()
    parser.add_argument('search_path', help='Path to search folder with single locales .json files')
    args = parser.parse_args()
    search_path = os.path.abspath(args.search_path)

    # Path to JSON file with p12n settings. You can point to a local file if you
    # plan to run this script often (file is over 2 MB)
    # e.g. p12n_json_url = 'file://{0}'.format(os.path.abspath('./searchplugins.json'))
    p12n_json_url = 'https://l10n.mozilla-community.org/~flod/p12n/searchplugins.json'

    search_locales = []
    for locale in glob.glob(os.path.join(search_path, '*.json')):
        locale_name = os.path.basename(os.path.splitext(locale)[0])
        search_locales.append(locale_name)

    try:
        p12n_locales = []
        p12n_json = json.load(urllib2.urlopen(p12n_json_url))
        for locale in p12n_json['locales']:
            p12n_locales.append(locale)
    except Exception as e:
        print 'Error reading {0}:'.format(p12n_json_url), e

    # Analyze differences
    analyze_differences(p12n_locales, search_locales, p12n_json)

    # Create a list of locales to analyze (elements in common)
    common_locales = list(set(p12n_locales).intersection(search_locales))
    common_locales.sort()

    # Exclude en-US
    common_locales.remove('en-US')

    for locale in common_locales:
        # Map searchplugin names to filenames
        p12n_searchplugins = p12n_json['locales'][locale]['browser']['aurora']['searchplugins'];
        name_mapping = {}
        for sp_filename, sp_data in p12n_searchplugins.iteritems():
            name_mapping[sp_data['name']] = sp_filename
        name_mapping['DuckDuckGo'] = 'ddg'

        # Get searchplugins explicitly set in search_order
        search_order = p12n_json['locales'][locale]['browser']['aurora']['p12n']['searchorder'];
        ordered_searchplugins = []
        for x in range(1,10):
            search_index = str(x)
            if search_index in search_order:
                ordered_searchplugins.append(search_order[search_index])

        other_searchplugins = []
        for sp_filename, sp_data in p12n_searchplugins.iteritems():
            if sp_data['name'] not in ordered_searchplugins:
                other_searchplugins.append(sp_data['name'])

        # Add ddg manually to all locales and sort
        other_searchplugins.append('DuckDuckGo')
        # Case insensitive sort
        other_searchplugins = sorted(other_searchplugins, key=lambda s: s.lower())

        # Map shortNames to file names
        all_searchplugins = []
        for sp in ordered_searchplugins + other_searchplugins:
            all_searchplugins.append(name_mapping[sp])

        # Load local JSON file
        file_name = os.path.join(search_path, locale + '.json')
        with open(file_name) as data_file:
            local_data = json.load(data_file)
            local_searchplugins = local_data['settings']['default']['visibleDefaultEngines']
            if local_searchplugins != all_searchplugins:
                print '\nLocale: {0}'.format(locale)
                print 'There are differences.'
                print 'remote p12n:', ', '.join(all_searchplugins)
                print 'local file: ', ', '.join(local_searchplugins)

        #sys.exit(1)
if __name__ == '__main__':
    main()