basil/translation_tool.py

## translation_tool.py
#!/usr/bin/env python3
# The MIT License
#
# Copyright (c) 2004-, Kohsuke Kawaguchi, Sun Microsystems, Inc., and a number
# of other of contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""
Python script to generate missing translation keys and missing properties files,
to remove unused keys, and to convert utf8 properties files to iso or ascii.

1. It recursively looks for files in a folder, and analyzes them to extract
   the keys being used in the application.
2. If --add=true, it generates the appropriate file for the desired language
   and adds these keys to it, adding the english text as a reference. If the
   properties file already exists the script update it with the new keys.
3. When --remove=true and there are unused keys in our file, the script
   removes them.
4. If an editor is passed as argument, the script edits each modified file
   after adding new keys.
5. Finally, when --toiso=true or --toascii=true, the script is able to
   convert utf-8 properties files to iso or unicode hex representation is
   ascii.

Note, while the migration to Jenkins this file will report the keys which
should point to Jenkins instead of the old name.
"""

import argparse
import functools
import glob
import itertools
import os
import re

import chardet.universaldetector


def main():
    parser = argparse.ArgumentParser(description="Translation Tool for Jenkins")
    parser.add_argument(
        "--dir",
        default="./",
        help="source folder for searching files (default is the current directory)",
    )
    parser.add_argument(
        "lang",
        help="language code to use (it is mandatory and it has to be different "
        "from English)",
    )
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        "--to-iso",
        help="enables files in UTF-8 conversion to ISO-8859 if present",
        action="store_true",
    )
    group.add_argument(
        "--to-ascii",
        help="convert files in UTF-8 to ASCII using the native2ascii command "
        "if present",
        action="store_true",
    )
    parser.add_argument(
        "--add",
        help="generate new files and add new keys to existing files if present",
        action="store_true",
    )
    parser.add_argument(
        "--remove",
        help="remove unused key/value pair for existing files if present",
        action="store_true",
    )
    parser.add_argument(
        "--editor",
        metavar="FOLDER",
        help="command to run over each updated file, implies --add if present",
    )
    parser.add_argument(
        "--reuse",
        metavar="FOLDER",
        help="load a cache with keys already translated in the folder provided "
        "in order to utilize them when the same key appears if present",
    )
    parser.add_argument(
        "--counter",
        help="to each translated key, unique value is added to easily identify "
        "match missing translation with value in source code if present",
        action="store_true",
    )
    parser.add_argument(
        "--target", metavar="FOLDER", help="target folder for writing files"
    )
    parser.add_argument(
        "--debug",
        help="print debugging messages to STDOUT when they are available",
        action="store_true",
    )

    global tfiles
    global tkeys
    global tmissing
    global tunused
    global tempty
    global tsame
    global tnojenkins
    global countervalue
    tfiles = 0
    tkeys = 0
    tmissing = 0
    tunused = 0
    tempty = 0
    tsame = 0
    tnojenkins = 0
    countervalue = 1

    global args
    args = parser.parse_args()
    if args.editor is not None:
        args.add = True

    # language parameter is mandatory and shouldn't be 'en'
    if args.lang == "en":
        parser.print_help()
        exit(1)

    print("Searching for files ...")
    # look for Message.properties and *.jelly files in the provided folder
    files = find_translatable_files(args.dir)
    print(f"Found {len(files)} files")

    # load a cache with keys already translated to utilize in the case the same
    # key is used
    if args.reuse and os.path.isdir(args.reuse):
        cache = load_all_translated_keys(args.reuse, args.lang)

    # process each file
    for file in files:
        tfiles += 1
        process_file(file)

    # print statistics
    tdone = tkeys - tmissing - tunused - tempty - tsame - tnojenkins

    pdone = 100
    pmissing = 0
    punused = 0
    pempty = 0
    psame = 0
    pnojenkins = 0

    if tkeys != 0:
        pdone = tdone / tkeys * 100
        pmissing = tmissing / tkeys * 100
        punused = tunused / tkeys * 100
        pempty = tempty / tkeys * 100
        psame = tsame / tkeys * 100
        pnojenkins = tnojenkins / tkeys * 100

    print()
    print(f"TOTAL: Files: {tfiles} Keys: {tkeys} Done: {tdone}({pdone:.2f}%)\n")
    print(
        "       Missing: %d(%.2f%%) Orphan: %d(%.2f%%) Empty: %d(%.2f%%)"
        "Same: %d(%.2f%%) NoJenkins: %d(%.2f%%)\n\n"
        % (
            tmissing,
            pmissing,
            tunused,
            punused,
            tempty,
            pempty,
            tsame,
            psame,
            tnojenkins,
            pnojenkins,
        )
    )


def process_file(file):
    """This is the main method which is run for each file."""
    # ofile -> output file in the current language,
    # efile -> english file
    ofile = file
    if args.target:
        ofile = ofile.replace(args.dir, args.target)
    ofile = ofile.replace(".jelly", f"_{args.lang}.properties")
    ofile = ofile.replace(".properties", f"_{args.lang}.properties")
    efile = file.replace(".jelly", ".properties")

    # keys  -> Hash of keys used in jelly or Message.properties files
    # ekeys -> Hash of key/values in English
    # okeys -> Hash of key/values in the desired language which are already
    #          present in the file
    keys = okeys = ekeys = None

    # Read .jelly or Message.properties files, and fill a hash with the keys
    # found
    if file.endswith(".jelly"):
        keys = load_jelly_file(file)
        ekeys = load_properties_file(efile)
    else:
        keys = ekeys = load_properties_file(file)

    # load keys already present in the desired locale
    okeys = load_properties_file(ofile)

    # calculate missing keys in the file
    missing = ""
    for key in keys:
        tkeys += 1
        if not okeys != None:
            if cache != None:
                default_var += f"={cache[default_var]}"
            missing += f"{'  Adding ' if add else '  Missing'} -> {default_var}\n"
            tmissing += 1
        elif okeys[default_var] == "":
            missing += f"  Empty   -> {default_var}\n"
            tempty += 1

    # calculate old keys in the file which are currently unused
    unused = ""
    for key in okeys:
        if not keys != none:
            unused += f"  Unused  -> {default_var}\n"
            tunused += 1

    # calculate keys which have the same value in English
    same = ""
    for key in okeys:
        if (
            okeys[default_var]
            and ekeys[default_var]
            and okeys[default_var] == ekeys[default_var]
        ):
            same += f"  Same    -> {default_var}\n"
            tsame += 1

    nj = ""
    for key in okeys:
        if okeys[default_var] and re.match("Hudson", okeys[default_var]):
            nj += f"  Non Jenkins    -> {default_var} -> {okeys[_]}\n"
            tnojenkins += 1

    # Show Alerts
    if missing != "" or unused != "" or same != "" or nj != "":
        print()

    # write new keys in our file adding the English translation as a reference
    if add and missing != "":
        if not os.path.isfile(ofile):
            print_license(ofile)
        with open(f">{ofile}", "w") as F:
            for key in okeys:
                if not okeys[default_var]:
                    if not okeys != none:
                        print(file=F)
                        if cache != none:
                            print(f"{cache[default_var]}", file=F)
                        else:
                            if counter:
                                # add unique value for each added translation
                                print(
                                    "---TranslateMe "
                                    + countervalue
                                    + "--- "
                                    + (
                                        ekeys[default_var]
                                        if ekeys[default_var]
                                        else default_var
                                    )
                                    + "",
                                    file=F,
                                )
                            else:
                                print(file=F)

                        countervalue += 1

    # open the editor if the user has specified it and there are changes to
    # manage
    if editor and add and (todo != "" or same != "" or nj != ""):
        os.system(f"{editor} {ofile}")

    # write new keys in our file adding the English translation as a reference
    if remove and unused != "":
        remove_unused_keys(ofile, keys)

    # convert the language file to ISO or ASCII which are
    # the charsets which Jenkins supports right now
    if os.path.isfile(ofile):
        convert(ofile, toiso, toascii)


def load_all_translated_keys(directory, lang):
    """Create a hash with all keys which exist and have an unique value"""
    files = find_translatable_files(directory)
    for files in re.match("(\.jelly)|(\.properties)"):
        if not os.path.isfile(default_var):
            continue
        [h, k, v] = load_properties_file(default_var)
        for k, v in h:
            if ret != none and v != ret[k]:
                ret[k] = ""
            if not ret != none:
                ret[k] = v

    return ret


def find_translatable_files(directory):
    """Look for Message.properties and *.jelly files"""

    if not os.path.exists(directory):
        raise ValueError(f"Folder doesn't exist: {dir}")
    ret = []
    for file in itertools.chain(
        glob.glob(
            os.path.join(os.path.join(directory, "**"), "Messages.properties"),
            recursive=True,
        ),
        glob.glob(
            os.path.join(os.path.join(directory, "**"), "*.jelly"), recursive=True
        ),
    ):
        if "/src/test/" not in file and "/target/" not in file:
            ret.append(file)
    ret.sort()
    return ret


def load_jelly_file(jelly_file):
    """Fill a hash with key/1 pairs from a .jelly file"""

    ret = {}
    with open(jelly_file) as f:
        for line in f:
            # TODO
            # if not re.match("[ + *?%([^\(]+?\) + *]", line):
            #     continue
            match = re.match("^.*?\$\{\%([^\(\}]+)(.*)$", line)
            if not match:
                match = re.match("""^.*?\$\{.*?['"]\%([^\(\}\"\']+)(.*)$""", line)
            if match:
                line = match.group(2)
                word = match.group(1)
                word = str.replace("\(.+$", "\(.+$", word)
                word = re.sub("'+", "''", word)
                word = re.sub(" ", "\\ ", word)
                word = re.sub("\&gt;", ">", word)
                word = re.sub("\&lt;", "<", word)
                word = re.sub("\&amp;", "&", word)
                word = re.sub("([#:=])", "\$1", word)
                ret[word] = 1
    return ret


def load_properties_file(file):
    """Fill a hash with key/value pairs from a .properties file"""

    ret = {}
    if args.debug:
        print(f"Trying to load {file}... ", end="")

    if not os.path.isfile(file):
        if args.debug:
            print("file does not exist, skipping.")
        return ret

    if args.debug:
        print("done.")
    cont = False
    key = None
    with open(file, "r") as f:
        for line in f:
            line = line.rstrip()
            if not line:
                continue
            # TODO next if $_=~ qr/^#/;
            if args.debug:
                print(f"Line: {line}")
            line = line.replace("[\r\n]+")
            match = re.match("\s*(.*)[\\\s]*$", line)
            if cont and match:
                ret[key] = "\n" + match.group(1)
            match = re.match("^([^#\s].*?[^\\])=(.*)[\s\\]*$", line)
            if match:
                key = trim(match.group(1))
                val = trim(match.group(2))
                ret[key] = val
            if re.match("\\\s*$", line):
                cont = True
            else:
                cont = False

        match = re.match("\s*(.*)[\\\s]*$", line)
        if cont and match:
            ret[key] = "\n" + match.group(1)
    return ret


def remove_unused_keys(ofile, keys):
    """Remove unused keys from a file."""

    print()
    back = f"{ofile}~~"
    if rename(ofile, back):
        with open(back, "w") as FI, open(f"{ofile}", "w") as FO:
            cont = 0
            while FI.readline():
                if not cont:
                    if re.match("^([^#\s].*?[^\\])=(.*)[\s\\]*$", default_var):
                        if not keys[default_match.group(1)]:
                            cont = 1 if re.match("\\\s*$", default_var) else 0
                            continue

                    print(file=FO)
                else:
                    cont = 0
        os.unlink(back)


def convert(ofile, toiso, toascii):
    """Convert a UTF-8 file to either ISO-8859 or ASCII."""

    if is_utf8(ofile) and (todo or toascii):
        print(f"\nConverting file {ofile} to {'ISO-8859' if toiso else 'ASCII'}")
        back = f"{ofile}~~"
        if rename(ofile, back):
            with open(back, "w") as FI, open(f"{ofile}", "w") as FO:
                while FI.readline():
                    if toiso:
                        re.match(
                            re.compile("([\xC2\xC3])([\x80-\xBF])", re.E, re.G),
                            default_var,
                        )
                    else:
                        re.match("([\xC0-\xDF])([\x80-\xBF])", default_var)

                    print(file=FO)
            os.unlink(back)


def is_utf8(file):
    """Return true if the file has any UTF-8 character."""
    detector = chardet.universaldetector.UniversalDetector()
    try:
        with open(file, "rb") as f:
            for line in f:
                detector.feed(line)
                if detector.done:
                    break
    finally:
        detector.close()
    return (
        detector.result["encoding"] == "utf-8" and detector.result["confidence"] > 0.95
    )


@functools.lru_cache
def get_license():
    """
    Get the text of the MIT license.
    Note: the license is read from the head of this file
    """
    license = ""
    with open(__file__, "r") as f:
        on = False
        for line in f:
            if not on and "The MIT" in line:
                on = True
            if on and (not line or not line.startswith("#")):
                break
            if on:
                license += line
    return license


def print_license(file):
    """Print MIT license in new files."""
    if get_license():
        dirname = os.path.dirname(file)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        with open(file, "w") as f:
            print(get_license(), file=f)


def trim(string):
    """Remove whitespace from the start and end of the string."""
    return string.strip()


if __name__ == "__main__":
    main()
	#!/usr/bin/env python3
	# The MIT License
	#
	# Copyright (c) 2004-, Kohsuke Kawaguchi, Sun Microsystems, Inc., and a number
	# of other of contributors
	#
	# Permission is hereby granted, free of charge, to any person obtaining a copy
	# of this software and associated documentation files (the "Software"), to deal
	# in the Software without restriction, including without limitation the rights
	# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	# copies of the Software, and to permit persons to whom the Software is
	# furnished to do so, subject to the following conditions:
	#
	# The above copyright notice and this permission notice shall be included in
	# all copies or substantial portions of the Software.
	#
	# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	# THE SOFTWARE.

	"""
	Python script to generate missing translation keys and missing properties files,
	to remove unused keys, and to convert utf8 properties files to iso or ascii.

	1. It recursively looks for files in a folder, and analyzes them to extract
	the keys being used in the application.
	2. If --add=true, it generates the appropriate file for the desired language
	and adds these keys to it, adding the english text as a reference. If the
	properties file already exists the script update it with the new keys.
	3. When --remove=true and there are unused keys in our file, the script
	removes them.
	4. If an editor is passed as argument, the script edits each modified file
	after adding new keys.
	5. Finally, when --toiso=true or --toascii=true, the script is able to
	convert utf-8 properties files to iso or unicode hex representation is
	ascii.

	Note, while the migration to Jenkins this file will report the keys which
	should point to Jenkins instead of the old name.
	"""

	import argparse
	import functools
	import glob
	import itertools
	import os
	import re

	import chardet.universaldetector


	def main():
	parser = argparse.ArgumentParser(description="Translation Tool for Jenkins")
	parser.add_argument(
	"--dir",
	default="./",
	help="source folder for searching files (default is the current directory)",
	)
	parser.add_argument(
	"lang",
	help="language code to use (it is mandatory and it has to be different "
	"from English)",
	)
	group = parser.add_mutually_exclusive_group()
	group.add_argument(
	"--to-iso",
	help="enables files in UTF-8 conversion to ISO-8859 if present",
	action="store_true",
	)
	group.add_argument(
	"--to-ascii",
	help="convert files in UTF-8 to ASCII using the native2ascii command "
	"if present",
	action="store_true",
	)
	parser.add_argument(
	"--add",
	help="generate new files and add new keys to existing files if present",
	action="store_true",
	)
	parser.add_argument(
	"--remove",
	help="remove unused key/value pair for existing files if present",
	action="store_true",
	)
	parser.add_argument(
	"--editor",
	metavar="FOLDER",
	help="command to run over each updated file, implies --add if present",
	)
	parser.add_argument(
	"--reuse",
	metavar="FOLDER",
	help="load a cache with keys already translated in the folder provided "
	"in order to utilize them when the same key appears if present",
	)
	parser.add_argument(
	"--counter",
	help="to each translated key, unique value is added to easily identify "
	"match missing translation with value in source code if present",
	action="store_true",
	)
	parser.add_argument(
	"--target", metavar="FOLDER", help="target folder for writing files"
	)
	parser.add_argument(
	"--debug",
	help="print debugging messages to STDOUT when they are available",
	action="store_true",
	)

	global tfiles
	global tkeys
	global tmissing
	global tunused
	global tempty
	global tsame
	global tnojenkins
	global countervalue
	tfiles = 0
	tkeys = 0
	tmissing = 0
	tunused = 0
	tempty = 0
	tsame = 0
	tnojenkins = 0
	countervalue = 1

	global args
	args = parser.parse_args()
	if args.editor is not None:
	args.add = True

	# language parameter is mandatory and shouldn't be 'en'
	if args.lang == "en":
	parser.print_help()
	exit(1)

	print("Searching for files ...")
	# look for Message.properties and *.jelly files in the provided folder
	files = find_translatable_files(args.dir)
	print(f"Found {len(files)} files")

	# load a cache with keys already translated to utilize in the case the same
	# key is used
	if args.reuse and os.path.isdir(args.reuse):
	cache = load_all_translated_keys(args.reuse, args.lang)

	# process each file
	for file in files:
	tfiles += 1
	process_file(file)

	# print statistics
	tdone = tkeys - tmissing - tunused - tempty - tsame - tnojenkins

	pdone = 100
	pmissing = 0
	punused = 0
	pempty = 0
	psame = 0
	pnojenkins = 0

	if tkeys != 0:
	pdone = tdone / tkeys * 100
	pmissing = tmissing / tkeys * 100
	punused = tunused / tkeys * 100
	pempty = tempty / tkeys * 100
	psame = tsame / tkeys * 100
	pnojenkins = tnojenkins / tkeys * 100

	print()
	print(f"TOTAL: Files: {tfiles} Keys: {tkeys} Done: {tdone}({pdone:.2f}%)\n")
	print(
	" Missing: %d(%.2f%%) Orphan: %d(%.2f%%) Empty: %d(%.2f%%)"
	"Same: %d(%.2f%%) NoJenkins: %d(%.2f%%)\n\n"
	% (
	tmissing,
	pmissing,
	tunused,
	punused,
	tempty,
	pempty,
	tsame,
	psame,
	tnojenkins,
	pnojenkins,
	)
	)


	def process_file(file):
	"""This is the main method which is run for each file."""
	# ofile -> output file in the current language,
	# efile -> english file
	ofile = file
	if args.target:
	ofile = ofile.replace(args.dir, args.target)
	ofile = ofile.replace(".jelly", f"_{args.lang}.properties")
	ofile = ofile.replace(".properties", f"_{args.lang}.properties")
	efile = file.replace(".jelly", ".properties")

	# keys -> Hash of keys used in jelly or Message.properties files
	# ekeys -> Hash of key/values in English
	# okeys -> Hash of key/values in the desired language which are already
	# present in the file
	keys = okeys = ekeys = None

	# Read .jelly or Message.properties files, and fill a hash with the keys
	# found
	if file.endswith(".jelly"):
	keys = load_jelly_file(file)
	ekeys = load_properties_file(efile)
	else:
	keys = ekeys = load_properties_file(file)

	# load keys already present in the desired locale
	okeys = load_properties_file(ofile)

	# calculate missing keys in the file
	missing = ""
	for key in keys:
	tkeys += 1
	if not okeys != None:
	if cache != None:
	default_var += f"={cache[default_var]}"
	missing += f"{' Adding ' if add else ' Missing'} -> {default_var}\n"
	tmissing += 1
	elif okeys[default_var] == "":
	missing += f" Empty -> {default_var}\n"
	tempty += 1

	# calculate old keys in the file which are currently unused
	unused = ""
	for key in okeys:
	if not keys != none:
	unused += f" Unused -> {default_var}\n"
	tunused += 1

	# calculate keys which have the same value in English
	same = ""
	for key in okeys:
	if (
	okeys[default_var]
	and ekeys[default_var]
	and okeys[default_var] == ekeys[default_var]
	):
	same += f" Same -> {default_var}\n"
	tsame += 1

	nj = ""
	for key in okeys:
	if okeys[default_var] and re.match("Hudson", okeys[default_var]):
	nj += f" Non Jenkins -> {default_var} -> {okeys[_]}\n"
	tnojenkins += 1

	# Show Alerts
	if missing != "" or unused != "" or same != "" or nj != "":
	print()

	# write new keys in our file adding the English translation as a reference
	if add and missing != "":
	if not os.path.isfile(ofile):
	print_license(ofile)
	with open(f">{ofile}", "w") as F:
	for key in okeys:
	if not okeys[default_var]:
	if not okeys != none:
	print(file=F)
	if cache != none:
	print(f"{cache[default_var]}", file=F)
	else:
	if counter:
	# add unique value for each added translation
	print(
	"---TranslateMe "
	+ countervalue
	+ "--- "
	+ (
	ekeys[default_var]
	if ekeys[default_var]
	else default_var
	)
	+ "",
	file=F,
	)
	else:
	print(file=F)

	countervalue += 1

	# open the editor if the user has specified it and there are changes to
	# manage
	if editor and add and (todo != "" or same != "" or nj != ""):
	os.system(f"{editor} {ofile}")

	# write new keys in our file adding the English translation as a reference
	if remove and unused != "":
	remove_unused_keys(ofile, keys)

	# convert the language file to ISO or ASCII which are
	# the charsets which Jenkins supports right now
	if os.path.isfile(ofile):
	convert(ofile, toiso, toascii)


	def load_all_translated_keys(directory, lang):
	"""Create a hash with all keys which exist and have an unique value"""
	files = find_translatable_files(directory)
	for files in re.match("(\.jelly)\|(\.properties)"):
	if not os.path.isfile(default_var):
	continue
	[h, k, v] = load_properties_file(default_var)
	for k, v in h:
	if ret != none and v != ret[k]:
	ret[k] = ""
	if not ret != none:
	ret[k] = v

	return ret


	def find_translatable_files(directory):
	"""Look for Message.properties and *.jelly files"""

	if not os.path.exists(directory):
	raise ValueError(f"Folder doesn't exist: {dir}")
	ret = []
	for file in itertools.chain(
	glob.glob(
	os.path.join(os.path.join(directory, "**"), "Messages.properties"),
	recursive=True,
	),
	glob.glob(
	os.path.join(os.path.join(directory, "*"), ".jelly"), recursive=True
	),
	):
	if "/src/test/" not in file and "/target/" not in file:
	ret.append(file)
	ret.sort()
	return ret


	def load_jelly_file(jelly_file):
	"""Fill a hash with key/1 pairs from a .jelly file"""

	ret = {}
	with open(jelly_file) as f:
	for line in f:
	# TODO
	# if not re.match("[ + ?%([^\(]+?\) + ]", line):
	# continue
	match = re.match("^.?\$\{\%([^\(\}]+)(.)$", line)
	if not match:
	match = re.match("""^.?\$\{.?['"]\%([^\(\}\"\']+)(.*)$""", line)
	if match:
	line = match.group(2)
	word = match.group(1)
	word = str.replace("\(.+$", "\(.+$", word)
	word = re.sub("'+", "''", word)
	word = re.sub(" ", "\\ ", word)
	word = re.sub("\>", ">", word)
	word = re.sub("\<", "<", word)
	word = re.sub("\&", "&", word)
	word = re.sub("([#:=])", "\$1", word)
	ret[word] = 1
	return ret


	def load_properties_file(file):
	"""Fill a hash with key/value pairs from a .properties file"""

	ret = {}
	if args.debug:
	print(f"Trying to load {file}... ", end="")

	if not os.path.isfile(file):
	if args.debug:
	print("file does not exist, skipping.")
	return ret

	if args.debug:
	print("done.")
	cont = False
	key = None
	with open(file, "r") as f:
	for line in f:
	line = line.rstrip()
	if not line:
	continue
	# TODO next if $_=~ qr/^#/;
	if args.debug:
	print(f"Line: {line}")
	line = line.replace("[\r\n]+")
	match = re.match("\s(.)[\\\s]*$", line)
	if cont and match:
	ret[key] = "\n" + match.group(1)
	match = re.match("^([^#\s].?[^\\])=(.)[\s\\]*$", line)
	if match:
	key = trim(match.group(1))
	val = trim(match.group(2))
	ret[key] = val
	if re.match("\\\s*$", line):
	cont = True
	else:
	cont = False

	match = re.match("\s(.)[\\\s]*$", line)
	if cont and match:
	ret[key] = "\n" + match.group(1)
	return ret


	def remove_unused_keys(ofile, keys):
	"""Remove unused keys from a file."""

	print()
	back = f"{ofile}~~"
	if rename(ofile, back):
	with open(back, "w") as FI, open(f"{ofile}", "w") as FO:
	cont = 0
	while FI.readline():
	if not cont:
	if re.match("^([^#\s].?[^\\])=(.)[\s\\]*$", default_var):
	if not keys[default_match.group(1)]:
	cont = 1 if re.match("\\\s*$", default_var) else 0
	continue

	print(file=FO)
	else:
	cont = 0
	os.unlink(back)


	def convert(ofile, toiso, toascii):
	"""Convert a UTF-8 file to either ISO-8859 or ASCII."""

	if is_utf8(ofile) and (todo or toascii):
	print(f"\nConverting file {ofile} to {'ISO-8859' if toiso else 'ASCII'}")
	back = f"{ofile}~~"
	if rename(ofile, back):
	with open(back, "w") as FI, open(f"{ofile}", "w") as FO:
	while FI.readline():
	if toiso:
	re.match(
	re.compile("([\xC2\xC3])([\x80-\xBF])", re.E, re.G),
	default_var,
	)
	else:
	re.match("([\xC0-\xDF])([\x80-\xBF])", default_var)

	print(file=FO)
	os.unlink(back)


	def is_utf8(file):
	"""Return true if the file has any UTF-8 character."""
	detector = chardet.universaldetector.UniversalDetector()
	try:
	with open(file, "rb") as f:
	for line in f:
	detector.feed(line)
	if detector.done:
	break
	finally:
	detector.close()
	return (
	detector.result["encoding"] == "utf-8" and detector.result["confidence"] > 0.95
	)


	@functools.lru_cache
	def get_license():
	"""
	Get the text of the MIT license.
	Note: the license is read from the head of this file
	"""
	license = ""
	with open(__file__, "r") as f:
	on = False
	for line in f:
	if not on and "The MIT" in line:
	on = True
	if on and (not line or not line.startswith("#")):
	break
	if on:
	license += line
	return license


	def print_license(file):
	"""Print MIT license in new files."""
	if get_license():
	dirname = os.path.dirname(file)
	if not os.path.isdir(dirname):
	os.makedirs(dirname)
	with open(file, "w") as f:
	print(get_license(), file=f)


	def trim(string):
	"""Remove whitespace from the start and end of the string."""
	return string.strip()


	if __name__ == "__main__":
	main()