Pkg-Config file parser (and tests)

pck_config_file.py

# coding=utf-8

import os
from collections import OrderedDict

from conans.errors import ConanException


class PkgConfigFile(object):
    """
    Represents the data stored in a pkg-config file.

    Keyword Fields:
    * Name (self.name)
        A human-readable name for the library or package. This does not affect usage of the
        pkg-config tool, which uses the name of the .pc file.
    * Description (self.description)
        A brief description of the package.
    * URL (self.url)
        An URL where people can get more information about and download the package.
    * Version (self.version)
        A string specifically defining the version of the package.
    * Requires (self.requires)
        A comma-separated list of packages required by this package. The versions of these packages
        may be specified using the comparison operators =, <, >, <= or >=.
    * Requires.private (self.requiresPrivate)
        A list of private packages required by this package but not exposed to
        applications. The version specific rules from the Requires field also apply here.
    * Conflicts (self.conflicts)
        An optional field describing packages that this one conflicts with. The version specific
        rules from the Requires field also apply here. This field also takes multiple instances
        of the same package. E.g., Conflicts: bar < 1.2.3, bar >= 1.3.0.
    * Cflags (self.cflags)
        The compiler flags specific to this package and any required libraries that don't support
        pkg-config. If the required libraries support pkg-config, they should be added to Requires
        or Requires.private.
    * Libs (self.libs)
        The link flags specific to this package and any required libraries that don't support
        pkg-config. The same rule as Cflags applies here.
    * Libs.private (self.libsPrivate)
        The link flags for private libraries required by this package but not exposed to
        applications. The same rule as Cflags applies here.

    Variables:
    * self.variables
    """

    def __init__(self):
        # Keyword fields
        self.name = None
        self.description = None
        self.url = None
        self.version = None
        self.requires = None
        self.requires_private = None
        self.conflicts = None
        self.cflags = None
        self.libs = None
        self.libs_private = None

        # Variables
        self.variables = OrderedDict()

    def load(self, filename):
        try:
            self._load(filename=filename)
        except Exception as e:
            raise ConanException("Error parsing pkg-config file '{}': {}".format(filename, e))

    def _join_continued_lines(self, lines):
        ret = []
        current_line = []
        for line in lines:
            ending_slashes = len(line) - len(line.rstrip('\\'))
            if ending_slashes % 2 == 1:
                current_line.append(line[:-1])
            else:
                current_line.append(line)
                ret.append(''.join(current_line))
                current_line.clear()
        return ret

    def _remove_comments(self, lines):
        for line in lines:
            without_comments = []
            chunks = line.split("#")
            if isinstance(chunks, list):
                chunks.append(None)
                for v, w in zip(chunks[:-1], chunks[1:]):
                    ending_slashes = len(v) - len(v.rstrip('\\'))
                    if ending_slashes % 2 == 1:
                        without_comments.append(v[:-1])
                        continue
                    else:
                        without_comments.append(v)
                        break
            else:
                without_comments.append(chunks)

            yield '#'.join(without_comments)

    def _load(self, filename):
        with open(filename) as reader:
            lines = [line.rstrip('\r\n') for line in reader.readlines()]
            lines = self._join_continued_lines(lines)
            lines = self._remove_comments(lines)

            # Once we have joined lines, we can parse them
            for raw_line in lines:
                line = raw_line.strip()

                if len(line) > 0:
                    iColon = line.find(":")
                    iEquals = line.find("=")

                    line_is_keyword = False  # if not a keyword, then it's a variable

                    if iColon == 0 or iEquals == 1:
                        raise Exception("line starts with ':' or '=' character: {}".format(raw_line))
                    if iColon == -1 and iEquals == -1:
                        raise Exception("line could not be interpreted: {}".format(raw_line))
                    elif iColon == -1:
                        line_is_keyword = False
                    elif iEquals == -1:
                        line_is_keyword = True
                    else:
                        line_is_keyword = iColon < iEquals

                    if line_is_keyword:
                        key = line[0:iColon]
                        value = line[iColon+1:].strip()
                    else:
                        key = line[0:iEquals]
                        value = line[iEquals+1:]

                    if line_is_keyword:
                        # Do not allow escaped comments in keywords
                        found = value.find("#")
                        if found != -1:
                            value = value[:found].strip()

                        if key == "Name":
                            self.name = value
                        elif key == "Description":
                            self.description = value
                        elif key == "URL":
                            self.url = value
                        elif key == "Version":
                            self.version = value
                        elif key == "Requires":
                            self.requires = value
                        elif key == "Requires.private":
                            self.requires_private = value
                        elif key == "Conflicts":
                            self.conflicts = value
                        elif key == "Cflags" or key == "CFlags":
                            self.cflags = value
                        elif key == "Libs":
                            self.libs = value
                        elif key == "Libs.private":
                            self.libs_private = value
                        else:
                            raise Exception("unrecognized keyword '{}' in line: {}".format(key,
                                                                                           raw_line))
                    else:
                        self.variables[key] = value

    def expand_variables(self, **variables):
        """ Do the variable expansion, values in `variables` takes precedence. """
        raise NotImplementedError

    def save(self, filename):
        # ensure the parent folder exists
        parent_folder = os.path.dirname(filename)
        os.makedirs(parent_folder, exist_ok=True)

        # open the file for writing
        with open(filename, "w") as writer:
            # write variables
            for key, value in self.variables.items():
                writer.write("{}={}\n".format(key, value))

            writer.write("\n")

            # write keywords
            if self.name is not None:
                writer.write("Name: {}\n".format(self.name))
            if self.description is not None:
                writer.write("Description: {}\n".format(self.description))
            if self.url is not None:
                writer.write("URL: {}\n".format(self.url))
            if self.version is not None:
                writer.write("Version: {}\n".format(self.version))
            if self.requires is not None:
                writer.write("Requires: {}\n".format(self.requires))
            if self.requires_private is not None:
                writer.write("Requires.private: {}\n".format(self.requires_private))
            if self.conflicts is not None:
                writer.write("Conflicts: {}\n".format(self.conflicts))
            if self.cflags is not None:
                writer.write("Cflags: {}\n".format(self.cflags))
            if self.libs is not None:
                writer.write("Libs: {}\n".format(self.libs))
            if self.libs_private is not None:
                writer.write("Libs.private: {}\n".format(self.libs_private))

    def __eq__(self, other):
        return all([self.variables == other.variables,
                    self.name == other.name,
                    self.description == other.description,
                    self.url == other.url,
                    self.version == other.version,
                    self.requires == other.requires,
                    self.requires_private == other.requires_private,
                    self.conflicts == other.conflicts,
                    self.cflags == other.cflags,
                    self.libs == other.libs,
                    self.libs_private == other.libs_private])
      

test_pck_config_file.py

# coding=utf-8

import os
import unittest
import tempfile
import shutil
import uuid
import subprocess

from conans.util.files import load
from conans.test.utils.tools import try_remove_readonly
from conans.client.wrappers.pkg_config_file import PkgConfigFile
from conans.client.tools.env import environment_append


def check_pkg_config():
    try:
        os.system("pkg-config --version")
    except Exception:
        return False
    else:
        return True


class PkgConfigFileBasicTest(unittest.TestCase):

    basic_content = """
        # Just an example file
        prefix=/usr # It will be substituted in the following vars.
        exec_prefix=${prefix}
        libdir=/usr/lib64
        includedir=${prefix}/include

        Name: FLAC
        Description: Free Lossless Audio Codec Library
        Version: 1.3.2
        Requires.private: ogg
        Libs: -L${libdir} -lFLAC
        Libs.private: -lm
        Cflags: -I${includedir}
    """

    def run(self, *args, **kwargs):
        self.tmp_folder = tempfile.mkdtemp(suffix='_conans')
        try:
            self.basic_content_file = os.path.join(self.tmp_folder, 'basic_content.pc')
            with open(self.basic_content_file, "w") as f:
                f.write(self.basic_content)

            super(PkgConfigFileBasicTest, self).run(*args, **kwargs)
        finally:
            shutil.rmtree(self.tmp_folder, onerror=try_remove_readonly)

    def test_basic_file(self):
        pkg_file = PkgConfigFile()
        pkg_file.load(self.basic_content_file)

        # Keywords
        self.assertEqual(pkg_file.name, "FLAC")
        self.assertEqual(pkg_file.description, "Free Lossless Audio Codec Library")
        self.assertEqual(pkg_file.version, "1.3.2")
        self.assertEqual(pkg_file.requires_private, "ogg")
        self.assertEqual(pkg_file.libs, "-L${libdir} -lFLAC")
        self.assertEqual(pkg_file.libs_private, "-lm")
        self.assertEqual(pkg_file.cflags, "-I${includedir}")

        # Vars
        self.assertEqual(pkg_file.variables["prefix"], "/usr")
        self.assertEqual(pkg_file.variables["exec_prefix"], "${prefix}")
        self.assertEqual(pkg_file.variables["libdir"], "/usr/lib64")
        self.assertEqual(pkg_file.variables["includedir"], "${prefix}/include")

    def test_equality(self):
        pkg1 = PkgConfigFile()
        pkg1.load(self.basic_content_file)

        pkg2 = PkgConfigFile()
        self.assertNotEqual(pkg1, pkg2)

        pkg2.load(self.basic_content_file)
        self.assertEqual(pkg1, pkg2)

        pkg2.version = "2.3.4"
        self.assertNotEqual(pkg1, pkg2)

    def test_idempotent(self):
        pkg_file = PkgConfigFile()
        pkg_file.load(self.basic_content_file)

        other_file = os.path.join(self.tmp_folder, str(uuid.uuid4()) + '.pc')
        pkg_file.save(other_file)

        pkg2_file = PkgConfigFile()
        pkg2_file.load(other_file)

        self.assertEqual(pkg_file, pkg2_file)

        another_file = os.path.join(self.tmp_folder, str(uuid.uuid4()) + '.pc')
        pkg2_file.save(another_file)
        self.assertNotEqual(other_file, another_file)
        self.assertEqual(load(other_file), load(another_file))


class PkgConfigFileComplexTest(unittest.TestCase):

    complex_content = r"""
        # This is a comment
        prefix=/usr# Also a comment
        my_name=FLAC 
        my_description=With a \# escaped \\\# it \\ is not \# a comment
        multiple_lines=Go \
for \
multiple \# lines # truncate \
on comment

        Name: FLAC
        Version: 0.1
        Description: AAA
        CFlags: Allow multiple \
lines in keywords too
        Libs: Do not allow \# escaped comments in keywords
    """

    def run(self, *args, **kwargs):
        self.tmp_folder = tempfile.mkdtemp(suffix='_conans')
        try:
            self.complex_content_file = os.path.join(self.tmp_folder, 'complex.pc')
            with open(self.complex_content_file, "w") as f:
                f.write(self.complex_content)

            super(PkgConfigFileComplexTest, self).run(*args, **kwargs)
        finally:
            shutil.rmtree(self.tmp_folder, onerror=try_remove_readonly)

    def test_parse(self):
        pkg_file = PkgConfigFile()
        pkg_file.load(self.complex_content_file)

        self.assertEqual(pkg_file.variables["prefix"], "/usr")
        self.assertEqual(pkg_file.variables["my_name"], "FLAC")
        self.assertEqual(pkg_file.variables["my_description"],
                         r"With a # escaped \\# it \\ is not # a comment")
        self.assertEqual(pkg_file.variables["multiple_lines"], "Go for multiple # lines")
        self.assertEqual(pkg_file.cflags, "Allow multiple lines in keywords too")
        self.assertEqual(pkg_file.libs, "Do not allow")

    @unittest.skipUnless(check_pkg_config(), "Requires pck-config executable")
    def test_output(self):
        pkg_file = PkgConfigFile()
        pkg_file.load(self.complex_content_file)

        def _call(command):
            cmd = ['pkg-config', ] + command.split(' ')
            return subprocess.check_output(cmd).decode().strip()

        with environment_append({"PKG_CONFIG_PATH": self.tmp_folder}):
            _call("--validate complex")

            self.assertEqual(_call("--variable=prefix complex"), pkg_file.variables["prefix"])
            self.assertEqual(_call("--variable=my_name complex"), pkg_file.variables["my_name"])
            self.assertEqual(_call("--variable=my_description complex"),
                             pkg_file.variables["my_description"])
            self.assertEqual(_call("--variable=multiple_lines complex"),
                             pkg_file.variables["multiple_lines"])
            self.assertEqual(_call("--cflags complex"), pkg_file.cflags)
            self.assertEqual(_call("--libs complex"), pkg_file.libs)