Check for markup

check_dtd_bug1539759.py

#!/usr/bin/env python3

import argparse
import codecs
import json
import logging
import os
import re
import six
import sys

logging.basicConfig()

try:
    from compare_locales import parser
except ImportError as e:
    print('FATAL: make sure that dependencies are installed')
    print(e)
    sys.exit(1)


class StringExtraction():

    def __init__(self):
        '''Initialize object.'''

        # Set defaults
        self.supported_formats = [
            '.dtd',
        ]

    def setLocale(self, locale):
        '''Set current locale.'''

        self.reference_locale = True if locale == 'en-US' else False
        self.locale = locale

    def setRepositoryPath(self, path):
        '''Set path to repository.'''

        # Strip trailing '/' from repository path
        self.repository_path = path.rstrip(os.path.sep)

    def extractFileList(self):
        '''Extract the list of supported files.'''

        self.file_list = []
        for root, dirs, files in os.walk(
                self.repository_path, followlinks=True):
            for file in files:
                for supported_format in self.supported_formats:
                    if file.endswith(supported_format):
                        self.file_list.append(os.path.join(root, file))
        self.file_list.sort()

    def getRelativePath(self, file_name):
        '''
        Get the relative path of a filename, prepend prefix_storage if
        defined.
        '''

        return file_name[len(self.repository_path) + 1:]

    def extractStrings(self):
        '''Extract strings from all files.'''

        # Create a list of files to analyze
        self.extractFileList()
        self.translations = {}

        for file_name in self.file_list:
            file_extension = os.path.splitext(file_name)[1]

            file_parser = parser.getParser(file_extension)
            file_parser.readFile(file_name)
            try:
                entities = file_parser.parse()
                for entity in entities:
                    # Ignore Junk
                    if isinstance(entity, parser.Junk):
                        #print('JUNK in {}\nFile: {}\nJunk: {}'.format(self.locale, file_name, entity))
                        continue
                    string_id = u'{0}:{1}'.format(
                        self.getRelativePath(file_name), six.text_type(entity))
                    self.translations[string_id] = entity.raw_val

                # Store reference strings
                if self.reference_locale:
                    self.getReferenceStringsMinor()
            except Exception as e:
                print('Error parsing file: {0}'.format(file_name))
                print(e)

    def getReferenceStringsMinor(self):
        '''Get a list of string IDs with <'''

        self.minor_strings = []
        self.reference_ids = self.translations.keys()
        for string_id, translation in self.translations.items():

            if '<' in translation:
                self.minor_strings.append(string_id)

    def checkIssues(self):
        '''Extract strings from all files.'''

        issues = []
        for string_id, translation in self.translations.items():
            if '<' not in translation:
                continue

            if string_id not in self.reference_ids:
                # Obsolete string
                continue

            if string_id not in self.minor_strings:
                issues.append('{}: {}'.format(string_id, translation))

        if issues:
            print('\n\nLocale: {}'.format(self.locale))
            print('\n'.join(issues))


def main():

    repos_path = '/Users/flodolo/mozilla/mercurial/l10n_clones/locales'
    locales = [x for x in os.listdir(repos_path) if not x.startswith('.')]

    ignored_locales = []

    locales = list(set(locales) - set(ignored_locales))
    locales.sort()

    # Initialize class
    extracted_strings = StringExtraction()

    # Extract strings for en-US, and keep them stored for comparison later
    print('Extracting reference en-US strings')
    gecko_string_path = '/Users/flodolo/mozilla/mercurial/gecko-strings-quarantine'
    extracted_strings.setLocale('en-US')
    extracted_strings.setRepositoryPath(gecko_string_path)
    extracted_strings.extractStrings()

    # Check other locales
    print('Checking other locales ({})'.format(len(locales)))
    for locale in locales:
        locale_path = os.path.join(repos_path, locale)
        extracted_strings.setRepositoryPath(locale_path)
        extracted_strings.setLocale(locale)
        extracted_strings.extractStrings()
        extracted_strings.checkIssues()


if __name__ == '__main__':
    main()

check_dtd_bug1539759.sh

#!/usr/bin/env bash

cd /Users/flodolo/mozilla/mercurial/l10n_clones/
source venv/bin/activate
compare-locales --version
python check_dtd_bug1539759.py