Investigating dkms failures from https://github.com/freedomofpress/securedrop-workstation/issues/590

kernel-debugging.py

#!/usr/bin/env python3
"""
Debugging script meant to reproduce the problems documented in
https://github.com/freedomofpress/securedrop-workstation/issues/590
"""
import logging
import subprocess
import sys
import os


logging.basicConfig(
    format="%(asctime)s %(levelname)-8s %(message)s",
    level=logging.DEBUG,
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("kernel-testing")

DEBS_DIR = "kernel-debs"
DEBS = {
    "old": [
        "linux-headers-4.14.169-grsec-workstation_4.14.169-grsec-workstation-1_amd64.deb",
        "linux-image-4.14.169-grsec-workstation_4.14.169-grsec-workstation-1_amd64.deb",
        "securedrop-workstation-grsec_4.14.169+buster_amd64.deb",
    ],
    "new": [
        "linux-headers-4.14.186-grsec-workstation_4.14.186-grsec-workstation-1_amd64.deb",
        "linux-image-4.14.186-grsec-workstation_4.14.186-grsec-workstation-1_amd64.deb",
        "securedrop-workstation-grsec_4.14.186+buster_amd64.deb",
    ],
}


def clean_modules():
    """
    Removes previously built versions of the
    u2mfn kernel module if present, to force its rebuild
    TODO: is the module missing sufficient to force a rebuild?
    """
    # Have only observed it for 4.14.186, not for 169. Still,
    # remove both if found
    cmd = [
        "sudo", "rm", "-f",
        "/usr/lib/modules/4.14.186-grsec-workstation/updates/dkms/u2mfn.ko",
        "/usr/lib/modules/4.14.169-grsec-workstation/updates/dkms/u2mfn.ko",
    ]
    subprocess.check_call(cmd)


def install_kernel_pkgs(version_target):
    """
    Installs a set of packages, 'old' or 'new'.
    """
    cmd = "sudo apt-get install -y --allow-downgrades".split()
    pkgs = [f"./{DEBS_DIR}/{x}" for x in DEBS[version_target]]
    cmd += pkgs
    try:
        with open(os.devnull, "w") as f:
            subprocess.check_call(cmd, stdout=f)
    except subprocess.CalledProcessError as e:
        # Unlikely to catch here, since apt exits zero even when dkms fails
        # https://github.com/freedomofpress/securedrop-debian-packaging/issues/184
        logger.error(f"Error reported by apt when installing {version_target} packages")
        logger.error(f"stderr: {e.stderr}")
        logger.error(f"stdout: {e.stdout}")


def ensure_no_problems():
    # Checkdd

    try:

        output = subprocess.check_output(["sudo", "grep", "FATAL", "/var/log/syslog"])
        logger.error("Encountered missing module in syslog")
        logger.error(f"Output was: {output}")
        raise Exception
    except subprocess.CalledProcessError:
        # logger.debug("No module errors")
        pass


if __name__ == "__main__":
    counter = 1
    log_every = 10
    logger.info(f"Beginning install loop, will report every {log_every} iterations")
    try:
        while True:
            if counter % 10 == 0:
                logger.info(f"Running reinstall attempt {counter}, no errors yet")
            for version_target in DEBS.keys():
                clean_modules()
                install_kernel_pkgs(version_target)
                ensure_no_problems()
            counter += 1
    except Exception as e:
        logger.error(f"Detected failure: {e} ; exiting")
        sys.exit(1)