salt.states.debconf module

apt.py

'''
Support for APT (Advanced Packaging Tool)
'''
# Import python libs
import os
import re

# Import Salt libs
import salt.utils


def __virtual__():
    '''
    Confirm this module is on a Debian based system
    '''
    return 'pkg' if __grains__['os'] in ('Debian', 'Ubuntu') else False


def __init__(opts):
    '''
    For Debian and derivative systems, set up
    a few env variables to keep apt happy and
    non-interactive.
    '''
    if __virtual__():
        env_vars = {
            'APT_LISTBUGS_FRONTEND': 'none',
            'APT_LISTCHANGES_FRONTEND': 'none',
            'DEBIAN_FRONTEND': 'noninteractive',
        }
        # Export these puppies so they persist
        os.environ.update(env_vars)


def available_version(name):
    '''
    Return the latest version of the named package available for upgrade or
    installation via the available apt repository

    CLI Example::

        salt '*' pkg.available_version <package name>
    '''
    version = ''
    cmd = 'apt-cache -q policy {0} | grep Candidate'.format(name)

    out = __salt__['cmd.run_stdout'](cmd)

    version_list = out.split()

    if len(version_list) >= 2:
        version = version_list[-1]

    return version


def version(name):
    '''
    Returns a string representing the package version or an empty string if not
    installed

    CLI Example::

        salt '*' pkg.version <package name>
    '''
    pkgs = list_pkgs(name)
    if name in pkgs:
        return pkgs[name]
    else:
        return ''


def refresh_db():
    '''
    Updates the APT database to latest packages based upon repositories

    Returns a dict::

        {'<database name>': Bool}

    CLI Example::

        salt '*' pkg.refresh_db
    '''
    cmd = 'apt-get -q update'

    out = __salt__['cmd.run_stdout'](cmd)

    servers = {}
    for line in out:
        cols = line.split()
        if not len(cols):
            continue
        ident = " ".join(cols[1:4])
        if 'Get' in cols[0]:
            servers[ident] = True
        else:
            servers[ident] = False

    return servers


def install(pkg, refresh=False, repo='', skip_verify=False,
            debconf=None, version=None, **kwargs):
    '''
    Install the passed package

    pkg
        The name of the package to be installed
    refresh : False
        Update apt before continuing
    repo : (default)
        Specify a package repository to install from
        (e.g., ``apt-get -t unstable install somepackage``)
    skip_verify : False
        Skip the GPG verification check (e.g., ``--allow-unauthenticated``)
    debconf : None
        Provide the path to a debconf answers file, processed before
        installation.
    version : None
        Install a specific version of the package, e.g. 1.0.9~ubuntu

    Return a dict containing the new package names and versions::

        {'<package>': {'old': '<old-version>',
                'new': '<new-version>']}

    CLI Example::

        salt '*' pkg.install <package name>
    '''
    salt.utils.daemonize_if(__opts__, **kwargs)
    if refresh:
        refresh_db()

    if debconf:
        __salt__['debconf.set_file'](debconf)

    ret_pkgs = {}
    old_pkgs = list_pkgs()

    if version:
        pkg = "{0}={1}".format(pkg, version)
    elif 'eq' in kwargs:
        pkg = "{0}={1}".format(pkg, kwargs['eq'])

    cmd = 'apt-get -q -y {confold}{verify}{target} install {pkg}'.format(
            confold=' -o DPkg::Options::=--force-confold',
            verify=' --allow-unauthenticated' if skip_verify else '',
            target=' -t {0}'.format(repo) if repo else '',
            pkg=pkg)

    __salt__['cmd.run'](cmd)
    new_pkgs = list_pkgs()

    for pkg in new_pkgs:
        if pkg in old_pkgs:
            if old_pkgs[pkg] == new_pkgs[pkg]:
                continue
            else:
                ret_pkgs[pkg] = {'old': old_pkgs[pkg],
                             'new': new_pkgs[pkg]}
        else:
            ret_pkgs[pkg] = {'old': '',
                         'new': new_pkgs[pkg]}

    return ret_pkgs


def remove(pkg):
    '''
    Remove a single package via ``apt-get remove``

    Returns a list containing the names of the removed packages.

    CLI Example::

        salt '*' pkg.remove <package name>
    '''
    ret_pkgs = []
    old_pkgs = list_pkgs()

    cmd = 'apt-get -q -y remove {0}'.format(pkg)
    __salt__['cmd.run'](cmd)
    new_pkgs = list_pkgs()
    for pkg in old_pkgs:
        if pkg not in new_pkgs:
            ret_pkgs.append(pkg)

    return ret_pkgs


def purge(pkg):
    '''
    Remove a package via ``apt-get purge`` along with all configuration
    files and unused dependencies.

    Returns a list containing the names of the removed packages

    CLI Example::

        salt '*' pkg.purge <package name>
    '''
    ret_pkgs = []
    old_pkgs = list_pkgs()

    # Remove inital package
    purge_cmd = 'apt-get -q -y purge {0}'.format(pkg)
    __salt__['cmd.run'](purge_cmd)

    new_pkgs = list_pkgs()

    for pkg in old_pkgs:
        if pkg not in new_pkgs:
            ret_pkgs.append(pkg)

    return ret_pkgs


def upgrade(refresh=True, **kwargs):
    '''
    Upgrades all packages via ``apt-get dist-upgrade``

    Returns a list of dicts containing the package names, and the new and old
    versions::

        [
            {'<package>':  {'old': '<old-version>',
                        'new': '<new-version>']
            }',
            ...
        ]

    CLI Example::

        salt '*' pkg.upgrade
    '''
    salt.utils.daemonize_if(__opts__, **kwargs)
    if refresh:
        refresh_db()

    ret_pkgs = {}
    old_pkgs = list_pkgs()
    cmd = 'apt-get -q -y -o DPkg::Options::=--force-confold dist-upgrade'
    __salt__['cmd.run'](cmd)
    new_pkgs = list_pkgs()

    for pkg in new_pkgs:
        if pkg in old_pkgs:
            if old_pkgs[pkg] == new_pkgs[pkg]:
                continue
            else:
                ret_pkgs[pkg] = {'old': old_pkgs[pkg],
                             'new': new_pkgs[pkg]}
        else:
            ret_pkgs[pkg] = {'old': '',
                         'new': new_pkgs[pkg]}

    return ret_pkgs


def list_pkgs(regex_string=""):
    '''
    List the packages currently installed in a dict::

        {'<package_name>': '<version>'}

    External dependencies::

        Virtual package resolution requires aptitude.
        Without aptitude virtual packages will be reported as not installed.

    CLI Example::

        salt '*' pkg.list_pkgs
        salt '*' pkg.list_pkgs httpd
    '''
    ret = {}
    cmd = 'dpkg-query --showformat=\'${{Status}} ${{Package}} ${{Version}}\n\' -W {0}'.format(regex_string)

    out = __salt__['cmd.run_stdout'](cmd)

    for line in out.split('\n'):
        cols = line.split()
        if len(cols) and ('install' in cols[0] or 'hold' in cols[0]) and 'installed' in cols[2]:
            ret[cols[3]] = cols[4]

    # If ret is empty at this point, check to see if the package is virtual.
    # We also need aptitude past this point.
    if not ret and __salt__['cmd.has_exec']('aptitude'):
        cmd = ('aptitude search "?name(^{0}$) ?virtual ?reverse-provides(?installed)"'
                .format(regex_string))

        out = __salt__['cmd.run_stdout'](cmd)
        if out:
            ret[regex_string] = '1' # Setting all 'installed' virtual package
                                    # versions to '1'

    return ret

def _get_upgradable():
    '''
    Utility function to get upgradable packages

    Sample return data:
    { 'pkgname': '1.2.3-45', ... }
    '''

    cmd = 'apt-get --just-print dist-upgrade'
    out = __salt__['cmd.run_stdout'](cmd)

    # rexp parses lines that look like the following:
    ## Conf libxfont1 (1:1.4.5-1 Debian:testing [i386])
    rexp = re.compile('(?m)^Conf '
                      '([^ ]+) ' # Package name
                      '\(([^ ]+) ' # Version
                      '([^ ]+)' # Release
                      '(?: \[([^\]]+)\])?\)$') # Arch
    keys = ['name', 'version', 'release', 'arch']
    _get = lambda l, k: l[keys.index(k)]

    upgrades = rexp.findall(out)

    r = {}
    for line in upgrades:
        name = _get(line, 'name')
        version = _get(line, 'version')
        r[name] = version

    return r

def list_upgrades():
    '''
    List all available package upgrades.

    CLI Example::

        salt '*' pkg.list_upgrades
    '''
    r = _get_upgradable()
    return r

def upgrade_available(name):
    '''
    Check whether or not an upgrade is available for a given package

    CLI Example::

        salt '*' pkg.upgrade_available <package name>
    '''
    r = name in _get_upgradable()
    return r

debconfmod.py

'''
Support for Debconf
'''
# Import Salt libs
import os
import re
import tempfile


def _unpack_lines(out):
    '''
    Unpack the debconf lines
    '''
    rexp = ('(?ms)'
            '^(?P<package>[^#]\S+)[\t ]+'
            '(?P<question>\S+)[\t ]+'
            '(?P<type>\S+)[\t ]+'
            '(?P<value>[^\n]*)$')
    lines = re.findall(rexp, out)
    return lines


def __virtual__():
    '''
    Confirm this module is on a Debian based system
    '''
    return 'debconf' if __grains__['os'] in ['Debian', 'Ubuntu'] else False


def get_selections(fetchempty=True):
    '''
    Answers to debconf questions for all packages in the following format::

        {'package': [['question', 'type', 'value'], ...]}

    CLI Example::

        salt '*' debconf.get_selections
    '''
    selections = {}
    cmd = 'debconf-get-selections'

    out = __salt__['cmd.run_stdout'](cmd)

    lines = _unpack_lines(out)

    for line in lines:
        package, question, type, value = line
        if fetchempty or value:
            (selections
                .setdefault(package, [])
                .append([question, type, value]))

    return selections


def show(name):
    '''
    Answers to debconf questions for a package in the following format::

        [['question', 'type', 'value'], ...]

    If debconf doesn't know about a package, we return None.

    CLI Example::

        salt '*' debconf.show <package name>
    '''

    result = None

    selections = get_selections()

    result = selections.get(name)
    return result

def _set_file(path):
    '''
    Execute the set selections command for debconf
    '''
    cmd = 'debconf-set-selections {0}'.format(path)

    __salt__['cmd.run_stdout'](cmd)

def set(package, question, type, value, *extra):
    '''
    Set answers to debconf questions for a package.

    CLI Example::

        salt '*' debconf.set <package> <question> <type> <value> [<value> ...]
    '''

    if extra:
        value = ' '.join((value,) + tuple(extra))

    fd, fname = tempfile.mkstemp(prefix="salt-")

    line = "{0} {1} {2} {3}".format(package, question, type, value)
    os.write(fd, line)
    os.close(fd)

    _set_file(fname)

    os.unlink(fname)

    return True

def set_file(path):
    '''
    Set answers to debconf questions from a file.

    CLI Example::

        salt '*' debconf.set_file salt://pathto/pkg.selections
    '''

    r = False

    path = __salt__['cp.cache_file'](path)
    if path:
        _set_file(path)
        r = True

    return r

debconfmod.py

'''
Management of Debconf settings
================================================

This module requires debconf-utils to be installed on a minon. Example usage:

.. code-block:: yaml

    locales:
      pkg.installed:
        - watch:
          - debconf: locales
      debconf.seed:
        - questions:
            locales/default_environment_locale:
              select: en_US.UTF-8
            locales/locales_to_be_generated:
              multiselect: 
                - en_US.UTF-8 UTF-8 
                - en_GB.UTF-8 UTF-8 

This will generate a file with the following preseed data that can be imported
with `debconf-set-selections`::

    locales locales/default_environment_locale      select          en_US.UTF-8
    locales locales/locales_to_be_generated         multiselect     en_US.UTF-8 UTF-8, en_GB.UTF-8 UTF-8

Instead of specifying all debconf data in SLS file it is possible to use
preseed file as jinja template to be loaded with `debconf-set-selections`:

.. code-block:: yaml

    slapd:
      debconf.seed_file:
        - ignore_passwords: true
        - source: salt://locales.selection
        - template: jinja
        - context:
        - defaults:

Among other data types accepted by debconf like `select`, `multiselect`,
`boolean`, `string`, `note`, `text` there is a `password` type. Packages use
them only during post-install stage to set passwords and then wipe them out.
Salt will by default put password data into debconf database only if the
package is not installed.

To prevent storing password data in debconf database it is strongly adviced to
make debconf state dependent on the relevant pkg state.

If you still want to store passwords in the debconf database you may set
`ignore_passwords` argument to `false`, however this will invoke all
`mod_watch` functions that depend on this state that will lead to continious
reconfiguring of the package that depends on debconf state.

One good way of updating passwords is using mod_watch function. When debconf
is called from this function, `ignore_passwords` option is set to `false` and
the password will be updated. Here is an extended example for mysql package:

.. code-block:: yaml
    mysql-server-5.5:
      pkg.installed:
        - require:
          - debconf: mysql-server-5.5
        - watch:
          - debconf: mysql-server-5.5
      mysql_user.present:
        - name: root
        - password: secret
        - watch_in:
          - debconf: mysql-server-5.5
      debconf.seed:
        - questions:
          mysql-server/root_password:
            password: secret
          mysql-server/root_password_again:
            password: secret
      
This example is useless since mysql_user state has already ways to change 
root passwords, but it should give the main idea.
'''

def __virtual__():
    '''
    Only load if debconf-get-selections is available
    '''
    if (__grains__['os'] in ['Debian', 'Ubuntu'] and
        __salt__['cmd.has_exec']('debconf-get-selections'):
        return 'debconf'

    else:
        return false

def seed_file(name, source, ignore_passwords=True, template=None, context=None, defaults=None):
    '''
    Ensures that the debconf values are set for the named package

    name
        The package that debconf settings are applied to

    source
        Path to the file containing debconf answers (can be generated via
        `debconf-get-selections`). 

    ignore_passwords
        Most packages store passwords in debconf database only during
        post-install stage and then wipe them out of debconf. While by default
        mismatching password fields won't initiate changes to be pushed to the
        database, this option if set to True allows to store password after 
        package installation. This, however, will invoke package
        reconfiguration only when executed inside watch call.

    template
        If this setting is applied then the named templating engine will be
        used to render the downloaded file, currently jinja, mako, and wempy
        are supported.

    context
        Context variables passed to the template.

    defaults
        Default values passed to the template.
    '''
    pass


def seed(name, questions, ignore_passwords=True):
    '''
    Ensures that the debconf values are set for the named package

    name
        The package that debconf settings are applied to

    questions
        Dictionary of debconf settings with items like `{question:{<type>:
        <value>}}`

    ignore_passwords
        Most packages store passwords in debconf database only during
        post-install stage and then wipe them out of debconf. While by default
        mismatching password fields won't initiate changes to be pushed to the
        database, this option if set to True allows to store password after 
        package installation. This, however, will invoke package
        reconfiguration only when executed inside watch call.
    '''
    pass

def mod_watch(name, **kwargs):
    pass

pkg.py

'''
Installation of packages using OS package managers such as yum or apt-get.
==========================================================================

Salt can manage software packages via the pkg state module, packages can be
set up to be installed, latest, removed and purged. Package management
declarations are typically rather simple:

.. code-block:: yaml

    vim:
      pkg.installed
'''
# Import python ilbs
import logging
import os
from distutils.version import LooseVersion

logger = logging.getLogger(__name__)


def __gen_rtag():
    '''
    Return the location of the refresh tag
    '''
    return os.path.join(__opts__['cachedir'], 'pkg_refresh')


def installed(
        name,
        version=None,
        refresh=False,
        repo='',
        skip_verify=False,
        **kwargs):
    '''
    Verify that the package is installed, and only that it is installed. This
    state will not upgrade an existing package and only verify that it is
    installed

    name
        The name of the package to install
    repo
        Specify a non-default repository to install from
    skip_verify : False
        Skip the GPG verification check for the package to be installed
    version : None
        Install a specific version of a package

    Usage::

        httpd:
          pkg:
            - installed
            - repo: mycustomrepo
            - skip_verify: True
            - version: 2.0.6~ubuntu3
    '''
    rtag = __gen_rtag()
    cver = __salt__['pkg.version'](name)
    if cver == version:
        # The package is installed and is the correct version
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': ('Package {0} is already installed and is the '
                            'correct version').format(name)}
    elif cver:
        # The package is installed
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package {0} is already installed'.format(name)}

    if __opts__['test']:
        return {'name': name,
                'changes': {},
                'result': None,
                'comment': 'Package {0} is set to be installed'.format(name)}

    if refresh or os.path.isfile(rtag):
        changes = __salt__['pkg.install'](name,
                          True,
                          version=version,
                          repo=repo,
                          skip_verify=skip_verify,
                          **kwargs)
        if os.path.isfile(rtag):
            os.remove(rtag)
    else:
        changes = __salt__['pkg.install'](name,
                          version=version,
                          repo=repo,
                          skip_verify=skip_verify,
                          **kwargs)
    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package {0} failed to install'.format(name)}
    return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package {0} installed'.format(name)}


def latest(name, refresh=False, repo='', skip_verify=False, **kwargs):
    '''
    Verify that the named package is installed and the latest available
    package. If the package can be updated this state function will update
    the package. Generally it is better for the ``installed`` function to be
    used, as ``latest`` will update the package whenever a new package is
    available.

    name
        The name of the package to maintain at the latest available version
    repo : (default)
        Specify a non-default repository to install from
    skip_verify : False
        Skip the GPG verification check for the package to be installed
    '''
    rtag = __gen_rtag()
    ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''}

    version = __salt__['pkg.version'](name)
    avail = __salt__['pkg.available_version'](name)

    if not version:
        # Net yet installed
        has_newer = True
    elif not avail:
        # Already at latest
        has_newer = False
    else:
        try:
            has_newer = LooseVersion(avail) > LooseVersion(version)
        except AttributeError:
            logger.debug(
                    'Error comparing versions for "{0}" ({1} > {2})'.format(
                    name, avail, version)
                    )
            ret['comment'] = 'No version could be retrieved for "{0}"'.format(
                    name)
            return ret

    if has_newer:
        if __opts__['test']:
            ret['result'] = None
            ret['comment'] = 'Package {0} is set to be upgraded'.format(name)
            return ret
        if refresh or os.path.isfile(rtag):
            ret['changes'] = __salt__['pkg.install'](name,
                             True,
                             repo=repo,
                             skip_verify=skip_verify,
                             **kwargs)
            if os.path.isfile(rtag):
                os.remove(rtag)

        else:
            ret['changes'] = __salt__['pkg.install'](name,
                             repo=repo,
                             skip_verify=skip_verify,
                             **kwargs)

        if ret['changes']:
            ret['comment'] = 'Package {0} upgraded to latest'.format(name)
            ret['result'] = True
        else:
            ret['comment'] = 'Package {0} failed to install'.format(name)
            ret['result'] = False
            return ret
    else:
        ret['comment'] = 'Package {0} already at latest'.format(name)
        ret['result'] = True

    return ret


def removed(name):
    '''
    Verify that the package is removed, this will remove the package via
    the remove function in the salt pkg module for the platform.

    name
        The name of the package to be removed
    '''
    changes = {}
    if not __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package {0} is not installed'.format(name)}
    else:
        if __opts__['test']:
            return {'name': name,
                    'changes': {},
                    'result': None,
                    'comment': 'Package {0} is set to be installed'.format(
                        name)}
        changes['removed'] = __salt__['pkg.remove'](name)
    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package {0} failed to remove'.format(name)}
    return {'name': name,
        'changes': changes,
        'result': True,
        'comment': 'Package {0} removed'.format(name)}


def purged(name):
    '''
    Verify that the package is purged, this will call the purge function in the
    salt pkg module for the platform.

    name
        The name of the package to be purged
    '''
    changes = {}
    if not __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package {0} is not installed'.format(name)}
    else:
        if __opts__['test']:
            return {'name': name,
                    'changes': {},
                    'result': None,
                    'comment': 'Package {0} is set to be purged'.format(name)}
        changes['removed'] = __salt__['pkg.purge'](name)

    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package {0} failed to purge'.format(name)}
    return {'name': name,
        'changes': changes,
        'result': True,
        'comment': 'Package {0} purged'.format(name)}


def mod_init(low):
    '''
    Set a flag to tell the install functions to refresh the package database.
    This ensures that the package database is refreshed only once durring
    a state run significaltly improving the speed of package management
    durring a state run.

    It sets a flag for a number of reasons, primarily due to timeline logic.
    When originally setting up the mod_init for pkg a number of corner cases
    arose with different package managers and how they refresh package data.
    '''
    if low['fun'] == 'installed' or low['fun'] == 'latest':
        rtag = __gen_rtag()
        if not os.path.exists(rtag):
            open(rtag, 'w+').write('')
        return True
    return False