Utility functions for testing Python importability

test_utils.py

import sys
import shutil
import tempfile
from os.path import dirname as p_dirname
from os.path import join as p_join
from types import GeneratorType
import unittest

import mock

from . import utils


THIS_DIR = p_dirname(__file__)
UPPER_DIR = p_dirname(THIS_DIR)
FAKE_MODULE_DIR = ('{}/some_module_structure'.format(THIS_DIR))


class PathToModulePathTestCase(unittest.TestCase):

    def test_abs_path(self):
        result = utils.path_to_module_name('/this/is/abs/path')
        self.assertEqual(result, 'this.is.abs.path')

    def test_abs_path_with_relative_to(self):
        result = utils.path_to_module_name('/this/is/abs/path', relative_to='/this')
        self.assertEqual(result, 'is.abs.path')

    def test_abs_path_with_relative_import(self):
        with self.assertRaises(ValueError):
            utils.path_to_module_name('/this/is/abs/path', relative_to='/that/foo/bar')

    def test_rel_path(self):
        result = utils.path_to_module_name('this/is/rel/path')
        self.assertEqual(result, 'this.is.rel.path')

    def test_rel_path_with_relative_to(self):
        result = utils.path_to_module_name('this/is/rel/path', relative_to='this')
        self.assertEqual(result, 'is.rel.path')

    def test_rel_path_with_relative_import(self):
        with self.assertRaises(ValueError):
            utils.path_to_module_name('this/is/abs/path', relative_to='that/foo/bar')

    def test_abs_path_with_ext(self):
        result = utils.path_to_module_name('/this/is/abs/file.foo')
        self.assertEqual(result, 'this.is.abs.file')

    def test_rel_path_with_ext(self):
        result = utils.path_to_module_name('this/is/rel/file.foo')
        self.assertEqual(result, 'this.is.rel.file')

    def test_abs_path_with_rel_relative_to(self):
        with self.assertRaises(ValueError):
            utils.path_to_module_name('/this/is/abs/path', 'this/is/rel')

    def test_rel_path_with_abs_relative_to(self):
        with self.assertRaises(ValueError):
            utils.path_to_module_name('this/is/rel/path', '/this/is/abs')


class FindPythonModuleTestCase(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_path_is_not_directory(self):
        with self.assertRaises(ValueError):
            utils.find_python_module('/some/not/exists/dir')

    def test_path(self):
        result = utils.find_python_module(FAKE_MODULE_DIR)

        self.assertEqual(len(result), 7)
        # Order does not matter, so use assertIn
        self.assertIn('some_module_structure', result)
        self.assertIn('some_module_structure.module_1', result)
        self.assertIn('some_module_structure.module_1.module_1_1', result)
        self.assertIn('some_module_structure.module_1.module_1_2', result)
        self.assertIn('some_module_structure.module_1.module_1_3', result)
        self.assertIn('some_module_structure.module_2', result)
        self.assertIn('some_module_structure.module_2.module_2_1', result)

    def test_path_with_trailing_slash(self):
        result = utils.find_python_module(FAKE_MODULE_DIR + '/')

        self.assertEqual(len(result), 7)
        # Order does not matter, so use assertIn
        self.assertIn('some_module_structure', result)
        self.assertIn('some_module_structure.module_1', result)
        self.assertIn('some_module_structure.module_1.module_1_1', result)
        self.assertIn('some_module_structure.module_1.module_1_2', result)
        self.assertIn('some_module_structure.module_1.module_1_3', result)
        self.assertIn('some_module_structure.module_2', result)
        self.assertIn('some_module_structure.module_2.module_2_1', result)

    def test_relative_to_upper_level_of_path(self):
        result = utils.find_python_module(FAKE_MODULE_DIR,
                                          relative_to=UPPER_DIR)

        self.assertEqual(len(result), 7)
        # Order does not matter, so use assertIn
        self.assertIn('test_importability.some_module_structure', result)
        self.assertIn('test_importability.some_module_structure.module_1', result)
        self.assertIn('test_importability.some_module_structure.module_1.module_1_1', result)
        self.assertIn('test_importability.some_module_structure.module_1.module_1_2', result)
        self.assertIn('test_importability.some_module_structure.module_1.module_1_3', result)
        self.assertIn('test_importability.some_module_structure.module_2', result)
        self.assertIn('test_importability.some_module_structure.module_2.module_2_1', result)

    def test_relative_to_cause_relative_import(self):
        with self.assertRaises(ValueError):
            utils.find_python_module(FAKE_MODULE_DIR,
                                     relative_to=p_join(FAKE_MODULE_DIR, 'module_1'))

    def test_directory_is_not_module(self):
        temp_dir_path = tempfile.mkdtemp()

        result = utils.find_python_module(temp_dir_path)

        self.assertEqual(len(result), 0)

        shutil.rmtree(temp_dir_path)


class TryImportTestCase(unittest.TestCase):

    def setUp(self):
        self.orig_sys_path = list(sys.path)
        # Adding the circular_imported_module into sys.path for importing
        sys.path.append(THIS_DIR)

    def tearDown(self):
        # Restore the sys.path
        sys.path = self.orig_sys_path

    def test_builtin_module(self):
        result = utils.try_import('os.path')

        self.assertEqual(result, (True, None))

    def test_not_exists_module(self):
        result = utils.try_import('some.weird.module')

        self.assertEqual(result[0], False)
        self.assertIn('ImportError: No module named some.weird.module', result[1])

    def test_circular_import_module(self):
        result = utils.try_import('circular_imported_module.a')

        self.assertEqual(result[0], False)
        self.assertIn('ImportError: cannot import name a', result[1])


class TryImportAllTestCase(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_all_valid_modules(self):
        results = utils.try_import_all(('os.path', 'sys', 'logging'))

        self.assertIsInstance(results, GeneratorType)
        self.assertEqual(list(results), [('os.path', True, None),
                                         ('sys', True, None),
                                         ('logging', True, None)])

    def test_some_invalid_modules(self):
        results = list(utils.try_import_all(('os.path', 'some.weird.module', 'logging')))

        self.assertEqual(results[0], ('os.path', True, None))

        self.assertEqual(results[1][0], 'some.weird.module')
        self.assertEqual(results[1][1], False)
        self.assertIn('ImportError', results[1][2])

        self.assertEqual(results[2], ('logging', True, None))


class LogImportAllMessagesTestCase(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    @mock.patch.object(utils, 'print')
    def test_log_results(self, m_print):
        utils.log_import_all_messages([('module_1', True, None),
                                       ('module_2', False, 'Some reason'),
                                       ('module_3', True, None)])
        m_print.assert_has_calls(
                [mock.call('Import module_1 succeeded', file=sys.stderr),
                 mock.call('FAILED to import module_2: Some reason', file=sys.stderr),
                 mock.call('Import module_3 succeeded', file=sys.stderr)])

    def test_log_results_to_file(self):
        file_obj = tempfile.TemporaryFile()
        utils.log_import_all_messages([('module_1', True, None),
                                       ('module_2', False, 'Some reason')],
                                      stream=file_obj)

        file_obj.flush()
        file_obj.seek(0)
        data = file_obj.read()
        self.assertEqual(data,
                         'Import module_1 succeeded\nFAILED to import module_2: Some reason\n')

        file_obj.close()

    @mock.patch.object(utils, 'print')
    def test_failed_only(self, m_print):
        utils.log_import_all_messages([('module_1', True, None),
                                       ('module_2', False, 'Some reason'),
                                       ('module_3', True, None)],
                                      failed_only=True)

        m_print.assert_has_calls(
                [mock.call('FAILED to import module_2: Some reason', file=sys.stderr)])

utils.py

from __future__ import print_function
import os
import sys
import traceback
from fnmatch import fnmatch


def path_to_module_name(path, relative_to=None):
    if os.path.isabs(path):
        relative_to = '/' if relative_to is None else relative_to
        if not os.path.isabs(relative_to):
            raise ValueError('Absolute path with relative relative_to')
        # XXX: In Python 2.6.6, relpath() does NOT function correctly on absolute path:
        # >>> os.path.relpath('/this/is/abs', '/')
        # '../this/is/abs'
        # So manually changing the path from abs to rel is required
        rel_path = os.path.normpath(path)[1:]
        rel_relative_to = os.path.normpath(relative_to)[1:]
        return path_to_module_name(rel_path, rel_relative_to)
    else:
        relative_to = '.' if relative_to is None else relative_to
        if os.path.isabs(relative_to):
            raise ValueError('Relative path with absolute relative_to')
        # The transformation happens here
        # XXX: The result returned by relpath() is normalized!
        final_path = os.path.relpath(path, relative_to)
        if final_path.find('..') != -1:
            raise ValueError('Relative import syntax is not supported')
        no_ext_final_path = os.path.splitext(final_path)[0]

        assert no_ext_final_path.find('.') == -1, "Path contains '.': {0}".format(no_ext_final_path)
        assert not no_ext_final_path.startswith('/'), "Path starts with '/': {0}".format(no_ext_final_path)
        assert not no_ext_final_path.endswith('/'), "Path ends with '/': {0}".format(no_ext_final_path)

        return no_ext_final_path.replace('/', '.')


def find_python_module(path, relative_to=None):
    """Find Python module from a directory. The returned module path is relative
    to the directory itself, not the whole path."""

    if not os.path.isdir(path):
        raise ValueError('{0} is not a directory'.format(path))

    relative_to = (os.path.dirname(os.path.normpath(path)) if relative_to is None
                   else relative_to)

    result = list()
    for dirpath, _, filenames in os.walk(path):
        if os.path.isfile(os.path.join(dirpath, '__init__.py')):
            # The dir itself is a Python module
            result.append(path_to_module_name(dirpath, relative_to))
        # Find all '*.py' files, except '__init__.py'
        python_files = (os.path.join(dirpath, filename)
                        for filename in filenames
                        if fnmatch(filename, '*.py') and filename != '__init__.py')
        module_names = (path_to_module_name(f, relative_to) for f in python_files)
        result.extend(module_names)
    return result


def try_import(name):
    """Try importing the name. Returning a tuple (result, traceback_message).

    If import succeeded, returns (True, None). Else return (False, traceback_message)
    where reason is a string.
    """
    try:
        imported_module = __import__(name, globals(), locals(), [], -1)
    except BaseException:
        return (False, traceback.format_exc())
    else:
        return (True, None)


def try_import_all(names):
    """Try import multiple names. Returning a list of tuple (name, result, reason).

    If importing the name succeeds, the tuple will be (name, True, None).
    Else the tuple will be (name, False, reason), where reason is a string
    """
    each_import_results = (try_import(name) for name in names)
    all_result = ((name, r[0], r[1]) for name, r
                  in zip(names, each_import_results))
    return all_result


def log_import_all_messages(results, stream=None, failed_only=False):
    """Helper function for logging the import results."""
    if stream is None:
        stream = sys.stderr

    for module_name, succeeded, fail_reason in results:
        if succeeded is False:
            print('FAILED to import {0}: {1}'.format(module_name, fail_reason),
                  file=stream)
        elif succeeded is True and failed_only is False:
            print('Import {0} succeeded'.format(module_name),
                  file=stream)